diff --git a/internal/pkg/additional/interface.go b/internal/pkg/additional/interface.go index 057d6b1d1..5ee8d1eaf 100644 --- a/internal/pkg/additional/interface.go +++ b/internal/pkg/additional/interface.go @@ -7,5 +7,5 @@ import ( ) type CollectorInterface interface { - AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) + AdditionalImagesCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) } diff --git a/internal/pkg/additional/local_stored_collector.go b/internal/pkg/additional/local_stored_collector.go index 32e52e253..c905f9c93 100644 --- a/internal/pkg/additional/local_stored_collector.go +++ b/internal/pkg/additional/local_stored_collector.go @@ -47,76 +47,81 @@ func (o LocalStorageCollector) destinationRegistry() string { // AdditionalImagesCollector - this looks into the additional images field // taking into account the mode we are in (mirrorToDisk, diskToMirror) // the image is downloaded in oci format -func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { - +func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { var allImages []v2alpha1.CopyImageSchema var allErrs []error + platformFilters := make(map[string][]string) o.Log.Debug(collectorPrefix+"setting copy option o.Opts.MultiArch=%s when collecting releases image", o.Opts.MultiArch) for _, img := range o.Config.ImageSetConfigurationSpec.Mirror.AdditionalImages { - var src, dest, tmpSrc, tmpDest string - - origin := img.Name - - imgSpec, err := image.ParseRef(img.Name) + src, dest, origin, err := o.resolveAdditionalImageSrcDest(img) if err != nil { - // OCPBUGS-33081 - skip if parse error (i.e semver and other) - o.Log.Warn("%v : SKIPPING", err) - allErrs = append(allErrs, fmt.Errorf("parse image %q: %w", img.Name, err)) + allErrs = append(allErrs, err) continue } + o.Log.Debug(collectorPrefix+"source %s", src) + o.Log.Debug(collectorPrefix+"destination %s", dest) - if img.TargetRepo != "" && !v2alpha1.IsValidPathComponent(img.TargetRepo) { - o.Log.Warn("invalid targetRepo %s for image %s : SKIPPING", img.TargetRepo, img.Name) - allErrs = append(allErrs, fmt.Errorf("invalid targetRepo %s for image %s", img.TargetRepo, img.Name)) - continue + allImages = append(allImages, v2alpha1.CopyImageSchema{ + Source: src, + Destination: dest, + Origin: origin, + Type: v2alpha1.TypeGeneric, + }) + if platforms := v2alpha1.ConvertPlatformsToStringSlice(img.Platforms); len(platforms) > 0 { + platformFilters[origin] = platforms } + } + cs := v2alpha1.CollectorSchema{AllImages: allImages} + if len(platformFilters) > 0 { + cs.PlatformFilters = platformFilters + } + return cs, errors.Join(allErrs...) +} - targetRepo := imgSpec.PathComponent - if img.TargetRepo != "" { - targetRepo = img.TargetRepo - } +// resolveAdditionalImageSrcDest validates an additional image entry and returns its +// resolved source, destination and origin references ready for mirroring. +func (o LocalStorageCollector) resolveAdditionalImageSrcDest(img v2alpha1.AdditionalImage) (src, dest, origin string, err error) { + origin = img.Name - targetTag := imgSpec.Tag - if img.TargetTag != "" { - targetTag = img.TargetTag - } + imgSpec, err := image.ParseRef(img.Name) + if err != nil { + // OCPBUGS-33081 - skip if parse error (i.e semver and other) + o.Log.Warn("%v : SKIPPING", err) + return "", "", "", fmt.Errorf("parse image %q: %w", img.Name, err) + } - switch { - case o.Opts.IsMirrorToDisk(), o.Opts.IsMirrorToMirror(): - tmpSrc, tmpDest = o.buildMirrorToDiskPaths(img, imgSpec, targetRepo, targetTag) - case o.Opts.IsDiskToMirror(), o.Opts.IsDelete(): - tmpSrc, tmpDest = o.buildDiskToMirrorPaths(img, imgSpec, targetRepo, targetTag) - } + if img.TargetRepo != "" && !v2alpha1.IsValidPathComponent(img.TargetRepo) { + o.Log.Warn("invalid targetRepo %s for image %s : SKIPPING", img.TargetRepo, img.Name) + return "", "", "", fmt.Errorf("invalid targetRepo %s for image %s", img.TargetRepo, img.Name) + } - if tmpSrc == "" || tmpDest == "" { - o.Log.Error(collectorPrefix+"unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name) - allErrs = append(allErrs, fmt.Errorf("unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name)) - continue - } + targetRepo, targetTag := resolveTargetRepoTag(img, imgSpec) - srcSpec, err := image.ParseRef(tmpSrc) // makes sure this ref is valid, and adds transport if needed - if err != nil { - o.Log.Error(errMsg, err.Error()) - allErrs = append(allErrs, fmt.Errorf("parse source %q: %w", tmpSrc, err)) - continue - } - src = srcSpec.ReferenceWithTransport - - destSpec, err := image.ParseRef(tmpDest) // makes sure this ref is valid, and adds transport if needed - if err != nil { - o.Log.Error(errMsg, err.Error()) - allErrs = append(allErrs, fmt.Errorf("parse destination %q: %w", tmpDest, err)) - continue - } - dest = destSpec.ReferenceWithTransport + var tmpSrc, tmpDest string + switch { + case o.Opts.IsMirrorToDisk(), o.Opts.IsMirrorToMirror(): + tmpSrc, tmpDest = o.buildMirrorToDiskPaths(img, imgSpec, targetRepo, targetTag) + case o.Opts.IsDiskToMirror(), o.Opts.IsDelete(): + tmpSrc, tmpDest = o.buildDiskToMirrorPaths(img, imgSpec, targetRepo, targetTag) + } - o.Log.Debug(collectorPrefix+"source %s", src) - o.Log.Debug(collectorPrefix+"destination %s", dest) + if tmpSrc == "" || tmpDest == "" { + o.Log.Error(collectorPrefix+"unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name) + return "", "", "", fmt.Errorf("unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name) + } - allImages = append(allImages, v2alpha1.CopyImageSchema{Source: src, Destination: dest, Origin: origin, Type: v2alpha1.TypeGeneric}) + srcSpec, err := image.ParseRef(tmpSrc) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return "", "", "", fmt.Errorf("parse source %q: %w", tmpSrc, err) + } + destSpec, err := image.ParseRef(tmpDest) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return "", "", "", fmt.Errorf("parse destination %q: %w", tmpDest, err) } - return allImages, errors.Join(allErrs...) + return srcSpec.ReferenceWithTransport, destSpec.ReferenceWithTransport, origin, nil } // buildMirrorToDiskPaths constructs source and destination paths for mirror-to-disk and mirror-to-mirror operations @@ -188,3 +193,17 @@ func (o LocalStorageCollector) buildDiskToMirrorPaths(img v2alpha1.AdditionalIma return tmpSrc, tmpDest } + +// resolveTargetRepoTag returns the effective target repository and tag for an additional image, +// applying any overrides from the image configuration. +func resolveTargetRepoTag(img v2alpha1.AdditionalImage, imgSpec image.ImageSpec) (string, string) { + targetRepo := imgSpec.PathComponent + if img.TargetRepo != "" { + targetRepo = img.TargetRepo + } + targetTag := imgSpec.Tag + if img.TargetTag != "" { + targetTag = img.TargetTag + } + return targetRepo, targetTag +} diff --git a/internal/pkg/additional/local_stored_collector_test.go b/internal/pkg/additional/local_stored_collector_test.go index 9fa031182..fa5d63594 100644 --- a/internal/pkg/additional/local_stored_collector_test.go +++ b/internal/pkg/additional/local_stored_collector_test.go @@ -107,7 +107,7 @@ func TestAdditionalImageCollector(t *testing.T) { log.Error(" %v ", err) t.Fatalf("should not fail") } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) // update opts @@ -148,7 +148,7 @@ func TestAdditionalImageCollector(t *testing.T) { log.Error(" %v ", err) t.Fatalf("should not fail") } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) t.Run("Testing AdditionalImagesCollector : diskToMirror with generateV1Tags should use latest for images by digest", func(t *testing.T) { @@ -187,7 +187,7 @@ func TestAdditionalImageCollector(t *testing.T) { log.Error(" %v ", err) t.Fatalf("should not fail") } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) // should error mirrorToDisk @@ -219,7 +219,7 @@ func TestAdditionalImageCollector(t *testing.T) { res, err := ex.AdditionalImagesCollector(ctx) // Should return error for images that failed to parse require.Error(t, err) - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) t.Run("Testing AdditionalImagesCollector : diskToMirror should collect valid images and return parse errors", func(t *testing.T) { @@ -250,7 +250,7 @@ func TestAdditionalImageCollector(t *testing.T) { res, err := ex.AdditionalImagesCollector(ctx) // Should return error for images that failed to parse require.Error(t, err) - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) } @@ -586,7 +586,7 @@ func TestAdditionalImageCollectorWithTargetRepoAndTag(t *testing.T) { } else { require.NoError(t, err) } - assert.ElementsMatch(t, c.expected, res) + assert.ElementsMatch(t, c.expected, res.AllImages) }) } } diff --git a/internal/pkg/api/v2alpha1/type_config.go b/internal/pkg/api/v2alpha1/type_config.go index 12b24c3f3..44d0b7149 100644 --- a/internal/pkg/api/v2alpha1/type_config.go +++ b/internal/pkg/api/v2alpha1/type_config.go @@ -94,7 +94,18 @@ type Platform struct { // Architectures defines one or more architectures // to mirror for the release image. This is defined at the // platform level to enable cross-channel upgrades. + // + // Deprecated: Use Platforms instead for fine-grained OS/Architecture control. + // + // For backward compatibility, if Platforms is empty and Architectures is set, + // it will be interpreted as linux/ for each architecture. Architectures []string `json:"architectures,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture release images. If empty and Architectures is empty, + // defaults to linux/amd64. This is defined at the platform level to enable + // cross-channel upgrades. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` // This new field will allow the diskToMirror functionality // to copy from a release location on disk Release string `json:"release,omitempty"` @@ -115,6 +126,9 @@ func (p Platform) DeepCopy() Platform { platformCopy.Architectures = make([]string, len(p.Architectures)) copy(platformCopy.Architectures, p.Architectures) + platformCopy.Platforms = make([]InstancePlatformFilter, len(p.Platforms)) + copy(platformCopy.Platforms, p.Platforms) + return platformCopy } @@ -183,6 +197,10 @@ type Operator struct { // path on disk for a template to use to complete catalogSource custom resource // generated by oc-mirror TargetCatalogSourceTemplate string `json:"targetCatalogSourceTemplate,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture catalog and operator images. If empty, mirrors all platforms. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` } // GetUniqueName determines the catalog name that will @@ -266,6 +284,10 @@ type Chart struct { // ImagePaths are custom JSON paths for images location // in the helm manifest or templates ImagePaths []string `json:"imagePaths,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture images referenced by this chart. If empty, mirrors all platforms. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` } // AdditionalImage contains image pull information for additional images, @@ -291,6 +313,10 @@ type AdditionalImage struct { // the image will be mirrored with the provided tag in the Name // field or a tag calculated from the partial digest. TargetTag string `json:"targetTag,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture images. If empty, mirrors all platforms. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` } // GetUniqueName determines the image name that will diff --git a/internal/pkg/api/v2alpha1/type_internal.go b/internal/pkg/api/v2alpha1/type_internal.go index 7374f5edd..d384b78d0 100644 --- a/internal/pkg/api/v2alpha1/type_internal.go +++ b/internal/pkg/api/v2alpha1/type_internal.go @@ -192,6 +192,9 @@ type CollectorSchema struct { AllImages []CopyImageSchema CopyImageSchemaMap CopyImageSchemaMap CatalogToFBCMap map[string]CatalogFilterResult // key is the mirror.operator.catalog + // PlatformFilters maps image origin to its platform filter list ("os/arch" strings). + // Populated by collectors; consumed by the batch worker to set InstancePlatforms per image. + PlatformFilters map[string][]string } type CopyImageSchemaMap struct { diff --git a/internal/pkg/api/v2alpha1/type_platform.go b/internal/pkg/api/v2alpha1/type_platform.go index 60e70e669..1d8e4159e 100644 --- a/internal/pkg/api/v2alpha1/type_platform.go +++ b/internal/pkg/api/v2alpha1/type_platform.go @@ -3,6 +3,7 @@ package v2alpha1 import ( "encoding/json" "errors" + "fmt" ) // DefaultPlatformArchitecture defines the default @@ -60,3 +61,50 @@ func (pt PlatformType) validate() error { } return nil } + +// InstancePlatformFilter defines OS and Architecture for filtering +// multi-architecture manifest lists. This allows selecting specific +// platforms (e.g., linux/amd64, linux/arm64) when mirroring images. +type InstancePlatformFilter struct { + // OS is the operating system (e.g., "linux", "windows") + OS string `json:"os"` + // Architecture is the CPU architecture (e.g., "amd64", "arm64", "ppc64le", "s390x") + Architecture string `json:"architecture"` +} + +// String returns the platform in "os/architecture" format (e.g., "linux/amd64") +func (p InstancePlatformFilter) String() string { + return p.OS + "/" + p.Architecture +} + +// Validate returns an error if OS or Architecture is empty. +func (p InstancePlatformFilter) Validate() error { + if p.OS == "" { + return fmt.Errorf("platform OS must not be empty (architecture: %q)", p.Architecture) + } + if p.Architecture == "" { + return fmt.Errorf("platform architecture must not be empty (OS: %q)", p.OS) + } + return nil +} + +// ConvertPlatformsToStringSlice converts a slice of InstancePlatformFilter to a slice +// of platform strings in "os/architecture" format. Returns nil if the input is empty. +// Entries with an empty OS or Architecture are skipped. +func ConvertPlatformsToStringSlice(platforms []InstancePlatformFilter) []string { + if len(platforms) == 0 { + return nil + } + + platformStrs := make([]string, 0, len(platforms)) + for _, p := range platforms { + if p.Validate() != nil { + continue + } + platformStrs = append(platformStrs, p.String()) + } + if len(platformStrs) == 0 { + return nil + } + return platformStrs +} diff --git a/internal/pkg/archive/archive.go b/internal/pkg/archive/archive.go index 790e09ba4..801557a2c 100644 --- a/internal/pkg/archive/archive.go +++ b/internal/pkg/archive/archive.go @@ -66,7 +66,7 @@ func NewMirrorArchive(opts *mirror.CopyOptions, destination, iscPath, workingDir // * docker/v2/blobs/sha256 : blobs that haven't been mirrored (diff) // * working-dir // * image set config -func (o *MirrorArchive) BuildArchive(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema) error { +func (o *MirrorArchive) BuildArchive(ctx context.Context, schema v2alpha1.CollectorSchema) error { if err := o.createTarball(); err != nil { return fmt.Errorf("unable to create the mirror archive: %w", err) } @@ -97,7 +97,7 @@ func (o *MirrorArchive) BuildArchive(ctx context.Context, collectedImages []v2al } // ignoring the error otherwise: continuing with an empty map in blobsInHistory - addedBlobs, err := o.addImagesDiff(ctx, collectedImages, blobsInHistory) + addedBlobs, err := o.addImagesDiff(ctx, schema, blobsInHistory) if err != nil { return fmt.Errorf("unable to add image blobs to the archive : %w", err) } @@ -136,10 +136,11 @@ func (o *MirrorArchive) createTarball() error { return nil } -func (o *MirrorArchive) addImagesDiff(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema, historyBlobs sets.Set[string]) (sets.Set[string], error) { +func (o *MirrorArchive) addImagesDiff(ctx context.Context, schema v2alpha1.CollectorSchema, historyBlobs sets.Set[string]) (sets.Set[string], error) { allAddedBlobs := sets.New[string]() - for _, img := range collectedImages { - imgBlobs, err := o.blobGatherer.GatherBlobs(ctx, img.Destination) + for _, img := range schema.AllImages { + allowedPlatforms := schema.PlatformFilters[img.Origin] + imgBlobs, err := o.blobGatherer.GatherBlobs(ctx, img.Destination, allowedPlatforms) var sigErr *SignatureBlobGathererError if err != nil && !errors.As(err, &sigErr) { return nil, fmt.Errorf("unable to find blobs corresponding to %s: %w", img.Destination, err) diff --git a/internal/pkg/archive/archive_test.go b/internal/pkg/archive/archive_test.go index a7adef36b..635a5df42 100644 --- a/internal/pkg/archive/archive_test.go +++ b/internal/pkg/archive/archive_test.go @@ -81,7 +81,7 @@ func TestArchive_BuildArchive(t *testing.T) { Origin: consts.DockerProtocol + "registry.redhat.io/ubi8/ubi:latest", }, } - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err != nil { t.Fatal(err) } @@ -104,7 +104,7 @@ func TestArchive_BuildArchive(t *testing.T) { Origin: consts.DockerProtocol + "registry.redhat.io/ubi8/ubi:latest", }, } - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err != nil { t.Fatal(err) } @@ -133,7 +133,7 @@ func TestArchive_CacheDirError(t *testing.T) { ma.cacheDir = "none" ma.workingDir = consts.TestFolder + "working-dir-fake" - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err == nil { t.Fatal("should fail") } @@ -158,7 +158,7 @@ func TestArchive_WorkingDirError(t *testing.T) { ma.cacheDir = consts.TestFolder + "cache-fake" ma.workingDir = "none" - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err == nil { t.Fatal("should fail") } @@ -182,7 +182,7 @@ func TestArchive_FileError(t *testing.T) { // force error for addFile ma.iscPath = "none" - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err == nil { t.Fatal("should fail") } @@ -463,7 +463,7 @@ func (ma *MirrorArchive) WithFakes(maxArchiveSize int64) (*MirrorArchive, error) return ma, nil } -func (mbg mockBlobGatherer) GatherBlobs(ctx context.Context, imgRef string) (sets.Set[string], error) { +func (mbg mockBlobGatherer) GatherBlobs(ctx context.Context, imgRef string, allowedPlatforms []string) (sets.Set[string], error) { blobs := sets.New( "sha256:2e39d55595ea56337b5b788e96e6afdec3db09d2759d903cbe120468187c4644", "sha256:94343313ec1512ab02267e4bc3ce09eecb01fda5bf26c56e2f028ecc72e80b18", diff --git a/internal/pkg/archive/image-blob-gatherer.go b/internal/pkg/archive/image-blob-gatherer.go index 24bb6c85c..2646b49c7 100644 --- a/internal/pkg/archive/image-blob-gatherer.go +++ b/internal/pkg/archive/image-blob-gatherer.go @@ -2,8 +2,10 @@ package archive import ( "context" + "encoding/json" "errors" "fmt" + "strings" digest "github.com/opencontainers/go-digest" "go.podman.io/image/v5/manifest" @@ -24,12 +26,13 @@ type ImageBlobGatherer struct { } type internalImageBlobGatherer struct { - imgRef string - sourceCtx *types.SystemContext - manifestBytes []byte - mimeType string - digest digest.Digest - copySignatures bool + imgRef string + sourceCtx *types.SystemContext + manifestBytes []byte + mimeType string + digest digest.Digest + copySignatures bool + allowedPlatforms []string // non-empty = only these os/arch pairs were mirrored (sparse) } func NewImageBlobGatherer(opts *mirror.CopyOptions, log clog.PluggableLoggerInterface) *ImageBlobGatherer { @@ -40,8 +43,9 @@ func NewImageBlobGatherer(opts *mirror.CopyOptions, log clog.PluggableLoggerInte } } -// GatherBlobs returns all container image blobs (including signature blobs if they exists). -func (o *ImageBlobGatherer) GatherBlobs(ctx context.Context, imgRef string) (blobs sets.Set[string], retErr error) { +// GatherBlobs returns all container image blobs (including signature blobs if they exist). +// See BlobsGatherer.GatherBlobs for the allowedPlatforms semantics. +func (o *ImageBlobGatherer) GatherBlobs(ctx context.Context, imgRef string, allowedPlatforms []string) (blobs sets.Set[string], retErr error) { // we are always gathering blobs from the local cache registry - skipping tls verification sourceCtx, err := o.opts.SrcImage.NewSystemContext() if err != nil { @@ -59,7 +63,7 @@ func (o *ImageBlobGatherer) GatherBlobs(ctx context.Context, imgRef string) (blo return nil, fmt.Errorf("error to get the digest of the image manifest %w", err) } - inImageBlogGather := internalImageBlobGatherer{imgRef: imgRef, sourceCtx: sourceCtx, manifestBytes: manifestBytes, mimeType: mime, digest: digest, copySignatures: !o.opts.RemoveSignatures} + inImageBlogGather := internalImageBlobGatherer{imgRef: imgRef, sourceCtx: sourceCtx, manifestBytes: manifestBytes, mimeType: mime, digest: digest, copySignatures: !o.opts.RemoveSignatures, allowedPlatforms: allowedPlatforms} if manifest.MIMETypeIsMultiImage(mime) { return o.multiArchBlobs(ctx, inImageBlogGather) @@ -93,13 +97,23 @@ func (o *ImageBlobGatherer) multiArchBlobs(ctx context.Context, in internalImage digests := manifestList.Instances() for _, digest := range digests { - blobs.Insert(digest.String()) singleIn := in singleIn.manifestBytes, singleIn.mimeType, err = o.ocmirrormanifest.ImageManifest(ctx, in.sourceCtx, in.imgRef, &digest) if err != nil { + if strings.Contains(err.Error(), "manifest unknown") && len(in.allowedPlatforms) > 0 { + platform := platformForDigest(in.manifestBytes, digest.String()) + if !platformAllowed(platform, in.allowedPlatforms) { + // Platform was intentionally not mirrored (sparse manifest filtering). + // Do not add its digest to blobs — the blob file does not exist locally. + o.log.Debug("Skipping absent platform %q for %s (not in allowed platforms)", platform, in.imgRef) + continue + } + } return nil, err } + // Only insert the platform digest after confirming the manifest is present. + blobs.Insert(digest.String()) singleIn.digest = digest singleArchBlobs, err := o.singleArchBlobs(ctx, singleIn) @@ -188,3 +202,36 @@ func (o *ImageBlobGatherer) imageSignatureBlobs(ctx context.Context, in internal return sigBlobs, nil } + +// platformForDigest returns the "os/arch" string for the manifest list entry +// matching digestStr, or an empty string if not found. +func platformForDigest(manifestBytes []byte, digestStr string) string { + var ml struct { + Manifests []struct { + Digest string `json:"digest"` + Platform struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } `json:"platform"` + } `json:"manifests"` + } + if err := json.Unmarshal(manifestBytes, &ml); err != nil { + return "" + } + for _, m := range ml.Manifests { + if m.Digest == digestStr { + return m.Platform.OS + "/" + m.Platform.Architecture + } + } + return "" +} + +// platformAllowed reports whether platform is in allowedPlatforms. +func platformAllowed(platform string, allowedPlatforms []string) bool { + for _, allowed := range allowedPlatforms { + if platform == allowed { + return true + } + } + return false +} diff --git a/internal/pkg/archive/image-blob-gatherer_test.go b/internal/pkg/archive/image-blob-gatherer_test.go index 469f3474a..57fba166a 100644 --- a/internal/pkg/archive/image-blob-gatherer_test.go +++ b/internal/pkg/archive/image-blob-gatherer_test.go @@ -221,7 +221,7 @@ func TestImageBlobGatherer_GatherBlobs(t *testing.T) { gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - blobs, err := gatherer.GatherBlobs(ctx, parentImage.destProtocol+u.Host+parentImage.dest) + blobs, err := gatherer.GatherBlobs(ctx, parentImage.destProtocol+u.Host+parentImage.dest, nil) if tc.expectedErrorType != nil { assert.True(t, reflect.TypeOf(err) == reflect.TypeOf(tc.expectedErrorType)) } else { @@ -257,7 +257,7 @@ func TestImageBlobGatherer_ImgRefError(t *testing.T) { } gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - _, err := gatherer.GatherBlobs(ctx, "error") + _, err := gatherer.GatherBlobs(ctx, "error", nil) assert.Equal(t, "invalid source name error: Invalid image name \"error\", expected colon-separated transport:reference", err.Error()) } @@ -289,7 +289,7 @@ func TestImageBlobGatherer_SrcContextError(t *testing.T) { } gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - _, err := gatherer.GatherBlobs(ctx, consts.DockerProtocol+"localhost/test:latest") + _, err := gatherer.GatherBlobs(ctx, consts.DockerProtocol+"localhost/test:latest", nil) assert.Contains(t, err.Error(), "pinging container registry localhost: Get \"http://localhost/v2/\": dial tcp [::1]:80: connect: connection refused") } @@ -328,6 +328,6 @@ func TestImageBlobGatherer_ImageSourceError(t *testing.T) { } gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - _, err = gatherer.GatherBlobs(ctx, consts.DockerProtocol+u.Host+"/bad-test:latest") + _, err = gatherer.GatherBlobs(ctx, consts.DockerProtocol+u.Host+"/bad-test:latest", nil) assert.Contains(t, err.Error(), "name unknown: Unknown name") } diff --git a/internal/pkg/archive/interface.go b/internal/pkg/archive/interface.go index fcb818e48..6a92d927b 100644 --- a/internal/pkg/archive/interface.go +++ b/internal/pkg/archive/interface.go @@ -9,11 +9,15 @@ import ( ) type BlobsGatherer interface { - GatherBlobs(ctx context.Context, imgRef string) (sets.Set[string], error) + // GatherBlobs returns all blobs for the given image. + // allowedPlatforms is the set of os/arch pairs that were intentionally mirrored + // (from CollectorSchema.PlatformFilters). Missing platforms outside this set are + // skipped; missing platforms inside it are real errors. + GatherBlobs(ctx context.Context, imgRef string, allowedPlatforms []string) (sets.Set[string], error) } type Archiver interface { - BuildArchive(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema) error + BuildArchive(ctx context.Context, schema v2alpha1.CollectorSchema) error } type UnArchiver interface { diff --git a/internal/pkg/batch/concurrent_chan_worker.go b/internal/pkg/batch/concurrent_chan_worker.go index 61f03dc3f..a08a93b0b 100644 --- a/internal/pkg/batch/concurrent_chan_worker.go +++ b/internal/pkg/batch/concurrent_chan_worker.go @@ -52,7 +52,8 @@ func (o *ChannelConcurrentBatch) Worker(ctx context.Context, collectorSchema v2a startTime := time.Now() copiedImages := v2alpha1.CollectorSchema{ - AllImages: []v2alpha1.CopyImageSchema{}, + AllImages: []v2alpha1.CopyImageSchema{}, + PlatformFilters: collectorSchema.PlatformFilters, } var errArray []mirrorErrorSchema @@ -144,8 +145,22 @@ func (o *ChannelConcurrentBatch) Worker(ctx context.Context, collectorSchema v2a options.PreserveDigests = false } + // If this image has specific platform requirements, use them + if platforms, ok := collectorSchema.PlatformFilters[img.Origin]; ok && len(platforms) > 0 { + options.InstancePlatforms = platforms + } + err = o.Mirror.Run(timeoutCtx, img.Source, img.Destination, mirror.Mode(opts.Function), &options) //nolint:contextcheck + // TODO + // TBD not sure if I like this approach below + // Single-arch images don't support platform filtering; retry without it. + if err != nil && options.InstancePlatforms != nil && strings.Contains(err.Error(), "no instances found for platform") { + o.Log.Debug("Platform filtering not supported for %s, retrying without filter", img.Source) + options.InstancePlatforms = nil + err = o.Mirror.Run(timeoutCtx, img.Source, img.Destination, mirror.Mode(opts.Function), &options) //nolint:contextcheck + } + switch { case err == nil: spinner.Increment() diff --git a/internal/pkg/cli/executor.go b/internal/pkg/cli/executor.go index b3bdd6925..49c7fb25d 100644 --- a/internal/pkg/cli/executor.go +++ b/internal/pkg/cli/executor.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "maps" "net" "net/http" "net/url" @@ -670,6 +671,10 @@ http: #htpasswd: #realm: basic-realm #path: /etc/registry +validation: + manifests: + indexes: + platforms: none ` var buff bytes.Buffer @@ -922,7 +927,7 @@ func (o *ExecutorSchema) RunMirrorToDisk(cmd *cobra.Command, args []string) erro } o.Log.Info(emoji.Package + " Preparing the tarball archive...") - return o.MirrorArchiver.BuildArchive(cmd.Context(), copiedSchema.AllImages) + return o.MirrorArchiver.BuildArchive(cmd.Context(), copiedSchema) } // RunMirrorToMirror - execute the mirror to mirror functionality @@ -1146,8 +1151,10 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche if len(o.Config.Mirror.Platform.Channels) > 0 || o.Config.Mirror.Platform.Release != "" || o.Config.Mirror.Platform.Graph { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting release images...") } + collectorSchema.PlatformFilters = make(map[string][]string) + // collect releases - releaseImgs, err := o.Release.ReleaseImageCollector(ctx) + releaseCS, err := o.Release.ReleaseImageCollector(ctx) if err != nil { if !o.Opts.IsDelete() { // Release image errors are fatal: we don't want to continue collection when they happen. @@ -1155,13 +1162,12 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche } releaseErr = err } - // exclude blocked images - releaseImgs = excludeImages(releaseImgs, o.Config.Mirror.BlockedImages) + releaseImgs := excludeImages(releaseCS.AllImages, o.Config.Mirror.BlockedImages) releaseImgs = removeDuplicatedImages(releaseImgs, o.Opts.Function) - collectorSchema.TotalReleaseImages = len(releaseImgs) o.Log.Debug(collecAllPrefix+"total release images to %s %d ", o.Opts.Function, collectorSchema.TotalReleaseImages) allRelatedImages = append(allRelatedImages, releaseImgs...) + maps.Copy(collectorSchema.PlatformFilters, releaseCS.PlatformFilters) if len(o.Config.Mirror.Operators) > 0 { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting operator images...") @@ -1171,45 +1177,51 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche if err != nil { operatorErr = err } else { - oImgs := operatorImgs.AllImages - // exclude blocked images - oImgs = excludeImages(oImgs, o.Config.Mirror.BlockedImages) + oImgs := excludeImages(operatorImgs.AllImages, o.Config.Mirror.BlockedImages) oImgs = removeDuplicatedImages(oImgs, o.Opts.Function) collectorSchema.TotalOperatorImages = len(oImgs) o.Log.Debug(collecAllPrefix+"total operator images to %s %d ", o.Opts.Function, collectorSchema.TotalOperatorImages) allRelatedImages = append(allRelatedImages, oImgs...) collectorSchema.CopyImageSchemaMap = operatorImgs.CopyImageSchemaMap collectorSchema.CatalogToFBCMap = operatorImgs.CatalogToFBCMap + + // TODO: if images are shared between types, maps.Copy will override platform filters for + // the same origin. Consider replacing with an append-based merge across all collectors: + // for k, v := range releaseCS.PlatformFilters { + // collectorSchema.PlatformFilters[k] = append(collectorSchema.PlatformFilters[k], v...) + // } + + maps.Copy(collectorSchema.PlatformFilters, operatorImgs.PlatformFilters) } if len(o.Config.Mirror.AdditionalImages) > 0 { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting additional images...") } // collect additionalImages - aImgs, err := o.AdditionalImages.AdditionalImagesCollector(ctx) + additionalCS, err := o.AdditionalImages.AdditionalImagesCollector(ctx) if err != nil { additionalImgErr = err } - // exclude blocked images - aImgs = excludeImages(aImgs, o.Config.Mirror.BlockedImages) + aImgs := excludeImages(additionalCS.AllImages, o.Config.Mirror.BlockedImages) aImgs = removeDuplicatedImages(aImgs, o.Opts.Function) collectorSchema.TotalAdditionalImages = len(aImgs) o.Log.Debug(collecAllPrefix+"total additional images to %s %d ", o.Opts.Function, collectorSchema.TotalAdditionalImages) allRelatedImages = append(allRelatedImages, aImgs...) + maps.Copy(collectorSchema.PlatformFilters, additionalCS.PlatformFilters) if len(o.Config.Mirror.Helm.Repositories) > 0 || len(o.Config.Mirror.Helm.Local) > 0 { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting helm images...") } - hImgs, err := o.HelmCollector.HelmImageCollector(ctx) + helmCS, err := o.HelmCollector.HelmImageCollector(ctx) if err != nil { helmErr = err } else { - // exclude blocked images - hImgs = excludeImages(hImgs, o.Config.Mirror.BlockedImages) + hImgs := excludeImages(helmCS.AllImages, o.Config.Mirror.BlockedImages) hImgs = removeDuplicatedImages(hImgs, o.Opts.Function) collectorSchema.TotalHelmImages = len(hImgs) o.Log.Debug(collecAllPrefix+"total helm images to %s %d ", o.Opts.Function, collectorSchema.TotalHelmImages) allRelatedImages = append(allRelatedImages, hImgs...) + maps.Copy(collectorSchema.PlatformFilters, helmCS.PlatformFilters) } sort.Sort(customsort.ByTypePriority(allRelatedImages)) diff --git a/internal/pkg/cli/executor_test.go b/internal/pkg/cli/executor_test.go index cddbbe837..95fb48e68 100644 --- a/internal/pkg/cli/executor_test.go +++ b/internal/pkg/cli/executor_test.go @@ -1245,9 +1245,9 @@ func (o *Collector) OperatorImageCollector(ctx context.Context) (v2alpha1.Collec return v2alpha1.CollectorSchema{AllImages: test}, nil } -func (o *Collector) ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *Collector) ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { if o.Fail { - return []v2alpha1.CopyImageSchema{}, fmt.Errorf("forced error release collector") + return v2alpha1.CollectorSchema{}, fmt.Errorf("forced error release collector") } test := []v2alpha1.CopyImageSchema{ {Source: "docker://registry/name/namespace/sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, @@ -1257,7 +1257,7 @@ func (o *Collector) ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyI {Source: "docker://registry/name/namespace/sometestimage-e@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, {Source: "docker://registry/name/namespace/sometestimage-f@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, } - return test, nil + return v2alpha1.CollectorSchema{AllImages: test}, nil } func (o *Collector) GraphImage() (string, error) { @@ -1268,9 +1268,9 @@ func (o *Collector) ReleaseImage(ctx context.Context) (string, error) { return "quay.io/openshift-release-dev/ocp-release:4.13.10-x86_64", nil } -func (o *Collector) AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *Collector) AdditionalImagesCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { if o.Fail { - return []v2alpha1.CopyImageSchema{}, fmt.Errorf("forced error additionalImages collector") + return v2alpha1.CollectorSchema{}, fmt.Errorf("forced error additionalImages collector") } test := []v2alpha1.CopyImageSchema{ {Source: "docker://registry/name/namespace/sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, @@ -1280,17 +1280,17 @@ func (o *Collector) AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.C {Source: "docker://registry/name/namespace/sometestimage-e@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, {Source: "docker://registry/name/namespace/sometestimage-f@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, } - return test, nil + return v2alpha1.CollectorSchema{AllImages: test}, nil } -func (o *Collector) HelmImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *Collector) HelmImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { if o.Fail { - return []v2alpha1.CopyImageSchema{}, fmt.Errorf("forced error helm collector") + return v2alpha1.CollectorSchema{}, fmt.Errorf("forced error helm collector") } - return []v2alpha1.CopyImageSchema{}, nil + return v2alpha1.CollectorSchema{}, nil } -func (o MockArchiver) BuildArchive(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema) error { +func (o MockArchiver) BuildArchive(ctx context.Context, schema v2alpha1.CollectorSchema) error { // return filepath.Join(o.destination, "mirror_000001.tar"), nil return nil } diff --git a/internal/pkg/config/defaults.go b/internal/pkg/config/defaults.go index 7f985c9e5..671d0c0e7 100644 --- a/internal/pkg/config/defaults.go +++ b/internal/pkg/config/defaults.go @@ -4,14 +4,19 @@ import ( "github.com/openshift/oc-mirror/v2/internal/pkg/api/v2alpha1" ) -// Complete set default values in the ImageSetConfiguration -// when applicable +// Complete sets default values in the ImageSetConfiguration when applicable. func Complete(cfg *v2alpha1.ImageSetConfiguration) { completeReleaseArchitectures(cfg) } func completeReleaseArchitectures(cfg *v2alpha1.ImageSetConfiguration) { - if len(cfg.Mirror.Platform.Channels) != 0 && len(cfg.Mirror.Platform.Architectures) == 0 { + // Only auto-fill the default architecture when neither Platforms nor Architectures is set. + // If Platforms is set, the Platforms branch handles everything; do not pollute Architectures. + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility + if len(cfg.Mirror.Platform.Channels) != 0 && + len(cfg.Mirror.Platform.Architectures) == 0 && + len(cfg.Mirror.Platform.Platforms) == 0 { + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility cfg.Mirror.Platform.Architectures = []string{v2alpha1.DefaultPlatformArchitecture} } } diff --git a/internal/pkg/config/validate.go b/internal/pkg/config/validate.go index c71e2fce7..d1d08dba1 100644 --- a/internal/pkg/config/validate.go +++ b/internal/pkg/config/validate.go @@ -17,7 +17,7 @@ type ( ) var ( - validationChecks = []validationFunc{validateOperatorOptions, validateReleaseChannels, validateBlockedImages} + validationChecks = []validationFunc{validateOperatorOptions, validateReleaseChannels, validateBlockedImages, validateReleasePlatformFields} validationDeleteChecks = []validationDeleteFunc{validateOperatorOptionsDelete, validateReleaseChannelsDelete} ) @@ -142,6 +142,19 @@ func validatePackageChannel(ctlgName string, pkg *v2alpha1.IncludePackage, ch *v return nil } +// validateReleasePlatformFields ensures that platform.platforms and platform.architectures +// are not used together — they are mutually exclusive. +func validateReleasePlatformFields(cfg *v2alpha1.ImageSetConfiguration) []error { + //nolint:staticcheck // SA1019: Architectures is deprecated but we check it for mutual exclusivity + if len(cfg.Mirror.Platform.Platforms) > 0 && len(cfg.Mirror.Platform.Architectures) > 0 { + return []error{fmt.Errorf( + "platform.platforms and platform.architectures cannot be used together; " + + "use platform.platforms for sparse manifest filtering (requires disabling manifest list blob checks on container registry side), or platform.architectures (deprecated) for OCP multi payload or single arch payloads", + )} + } + return nil +} + func validateReleaseChannels(cfg *v2alpha1.ImageSetConfiguration) []error { channels := sets.New[string]() for _, channel := range cfg.Mirror.Platform.Channels { diff --git a/internal/pkg/delete/delete_images_test.go b/internal/pkg/delete/delete_images_test.go index a328c2fe7..807467094 100644 --- a/internal/pkg/delete/delete_images_test.go +++ b/internal/pkg/delete/delete_images_test.go @@ -522,7 +522,7 @@ func (o mockBatch) Worker(ctx context.Context, collectorSchema v2alpha1.Collecto return collectorSchema, nil } -func (o *mockBlobs) GatherBlobs(ctx context.Context, image string) (sets.Set[string], error) { +func (o *mockBlobs) GatherBlobs(ctx context.Context, image string, allowedPlatforms []string) (sets.Set[string], error) { res := sets.New("sha256:95ad8395795ee0460baf05458f669d3b865535f213f015519ef9a221a6a08280") if o.Fail { return nil, fmt.Errorf("forced error") diff --git a/internal/pkg/helm/interface.go b/internal/pkg/helm/interface.go index 8b8211a90..0cc50ee3a 100644 --- a/internal/pkg/helm/interface.go +++ b/internal/pkg/helm/interface.go @@ -8,7 +8,7 @@ import ( ) type CollectorInterface interface { - HelmImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) + HelmImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) } type indexDownloader interface { diff --git a/internal/pkg/helm/local_stored_collector.go b/internal/pkg/helm/local_stored_collector.go index 4fff91415..cdac8a6fb 100644 --- a/internal/pkg/helm/local_stored_collector.go +++ b/internal/pkg/helm/local_stored_collector.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io/fs" + "maps" "net/http" "os" "path/filepath" @@ -78,123 +79,132 @@ func WithV1Tags(o CollectorInterface) CollectorInterface { return o } -func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { - var ( - allImages []v2alpha1.CopyImageSchema - allHelmImages []v2alpha1.RelatedImage - errs []error - ) +func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { + var allImages []v2alpha1.CopyImageSchema + var platformFilters map[string][]string + var errs []error switch { case lsc.Opts.IsMirrorToDisk() || lsc.Opts.IsMirrorToMirror(): defer lsc.cleanup() + allImages, platformFilters, errs = lsc.collectHelmImagesM2D() + case lsc.Opts.IsDiskToMirror(): + allImages, platformFilters, errs = lsc.collectHelmImagesD2M(o.generateV1DestTags) + } - var err error - imgs, errors := getHelmImagesFromLocalChart() - if len(errors) > 0 { - errs = append(errs, errors...) - } - if len(imgs) > 0 { - allHelmImages = append(allHelmImages, imgs...) - } + cs := v2alpha1.CollectorSchema{AllImages: allImages} + if len(platformFilters) > 0 { + cs.PlatformFilters = platformFilters + } + return cs, errors.Join(errs...) +} - for _, repo := range lsc.Config.Mirror.Helm.Repositories { - charts := repo.Charts +func (lsc *LocalStorageCollector) collectHelmImagesM2D() ([]v2alpha1.CopyImageSchema, map[string][]string, []error) { //nolint:cyclop + var allHelmImages []v2alpha1.RelatedImage + var errs []error + platformFilters := make(map[string][]string) - if err := repoAdd(repo); err != nil { + imgs, localPlatforms, errors := getHelmImagesFromLocalChart() + errs = append(errs, errors...) + if len(imgs) > 0 { + allHelmImages = append(allHelmImages, imgs...) + maps.Copy(platformFilters, localPlatforms) + } + + for _, repo := range lsc.Config.Mirror.Helm.Repositories { + charts := repo.Charts + if err := repoAdd(repo); err != nil { + errs = append(errs, err) + continue + } + if charts == nil { + var indexFile helmrepo.IndexFile + var err error + if indexFile, err = createIndexFile(repo.URL); err != nil { errs = append(errs, err) continue } - - if charts == nil { - var indexFile helmrepo.IndexFile - if indexFile, err = createIndexFile(repo.URL); err != nil { - errs = append(errs, err) - continue - } - - if charts, err = getChartsFromIndex("", indexFile); err != nil && charts == nil { - errs = append(errs, err) - continue - } - } - - for _, chart := range charts { - lsc.Log.Debug("Pulling chart %s", chart.Name) - ref := fmt.Sprintf("%s/%s", repo.Name, chart.Name) - dest := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) - path, _, err := lsc.Downloaders.chartDownloader.DownloadTo(ref, chart.Version, dest) - if err != nil { - errs = append(errs, err) - lsc.Log.Error("error pulling chart %s:%s", ref, err.Error()) - continue - } - - imgs, err := getImages(path, chart.ImagePaths...) - if err != nil { - errs = append(errs, err) - } - - allHelmImages = append(allHelmImages, imgs...) - + if charts, err = getChartsFromIndex("", indexFile); err != nil && charts == nil { + errs = append(errs, err) + continue } } - - allImages, err = prepareM2DCopyBatch(allHelmImages) - if err != nil { - lsc.Log.Error(errMsg, err.Error()) - errs = append(errs, err) - } - - case lsc.Opts.IsDiskToMirror(): - imgs, errors := getHelmImagesFromLocalChart() - if len(errors) > 0 { - errs = append(errs, errors...) - } - if len(imgs) > 0 { - allHelmImages = append(allHelmImages, imgs...) + for _, chart := range charts { + lsc.Log.Debug("Pulling chart %s", chart.Name) + ref := fmt.Sprintf("%s/%s", repo.Name, chart.Name) + dest := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) + path, _, err := lsc.Downloaders.chartDownloader.DownloadTo(ref, chart.Version, dest) + if err != nil { + errs = append(errs, err) + lsc.Log.Error("error pulling chart %s:%s", ref, err.Error()) + continue + } + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + chartImgs, err := getImages(path, chart.ImagePaths...) + if err != nil { + errs = append(errs, err) + } + allHelmImages = append(allHelmImages, chartImgs...) + for _, img := range chartImgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) + } } + } - for _, repo := range lsc.Config.Mirror.Helm.Repositories { - charts := repo.Charts - - if charts == nil { - var err error - if charts, err = getChartsFromIndex(repo.URL, helmrepo.IndexFile{}); err != nil { - errs = append(errs, err) - if charts == nil { - continue - } - } - } + allImages, err := prepareM2DCopyBatch(allHelmImages) + if err != nil { + lsc.Log.Error(errMsg, err.Error()) + errs = append(errs, err) + } + return allImages, platformFilters, errs +} - for _, chart := range charts { - chartDir := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) - path, err := resolveChartPath(chartDir, chart.Name, chart.Version) - if err != nil { - errs = append(errs, err) +func (lsc *LocalStorageCollector) collectHelmImagesD2M(generateV1Tags bool) ([]v2alpha1.CopyImageSchema, map[string][]string, []error) { + var allHelmImages []v2alpha1.RelatedImage + var errs []error + platformFilters := make(map[string][]string) + + imgs, localPlatforms, errors := getHelmImagesFromLocalChart() + errs = append(errs, errors...) + allHelmImages = append(allHelmImages, imgs...) + maps.Copy(platformFilters, localPlatforms) + + for _, repo := range lsc.Config.Mirror.Helm.Repositories { + charts := repo.Charts + if charts == nil { + var err error + if charts, err = getChartsFromIndex(repo.URL, helmrepo.IndexFile{}); err != nil { + errs = append(errs, err) + if charts == nil { continue } - - imgs, err := getImages(path, chart.ImagePaths...) - if err != nil { - errs = append(errs, err) - } - - allHelmImages = append(allHelmImages, imgs...) - } } - - var err error - allImages, err = prepareD2MCopyBatch(allHelmImages, o.generateV1DestTags) - if err != nil { - lsc.Log.Error(errMsg, err.Error()) - errs = append(errs, err) + for _, chart := range charts { + chartDir := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) + path, err := resolveChartPath(chartDir, chart.Name, chart.Version) + if err != nil { + errs = append(errs, err) + continue + } + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + chartImgs, err := getImages(path, chart.ImagePaths...) + if err != nil { + errs = append(errs, err) + } + allHelmImages = append(allHelmImages, chartImgs...) + for _, img := range chartImgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) + } } } - return allImages, errors.Join(errs...) + allImages, err := prepareD2MCopyBatch(allHelmImages, generateV1Tags) + if err != nil { + lsc.Log.Error(errMsg, err.Error()) + errs = append(errs, err) + } + return allImages, platformFilters, errs } func createTempFile(dir string) (func(), string, error) { @@ -226,11 +236,14 @@ func GetDefaultChartDownloader() chartDownloader { } } -func getHelmImagesFromLocalChart() ([]v2alpha1.RelatedImage, []error) { +func getHelmImagesFromLocalChart() ([]v2alpha1.RelatedImage, map[string][]string, []error) { var allHelmImages []v2alpha1.RelatedImage var errs []error + platformFilters := make(map[string][]string) for _, chart := range lsc.Config.Mirror.Helm.Local { + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + imgs, err := getImages(chart.Path, chart.ImagePaths...) if err != nil { errs = append(errs, err) @@ -238,10 +251,15 @@ func getHelmImagesFromLocalChart() ([]v2alpha1.RelatedImage, []error) { if len(imgs) > 0 { allHelmImages = append(allHelmImages, imgs...) + if len(platforms) > 0 { + for _, img := range imgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) + } + } } } - return allHelmImages, errs + return allHelmImages, platformFilters, errs } func repoAdd(chartRepo v2alpha1.Repository) error { @@ -590,7 +608,12 @@ func prepareM2DCopyBatch(images []v2alpha1.RelatedImage) ([]v2alpha1.CopyImageSc lsc.Log.Debug("source %s", src) lsc.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + }) } return result, nil } @@ -630,7 +653,12 @@ func prepareD2MCopyBatch(images []v2alpha1.RelatedImage, generateV1TagsFromDiges lsc.Log.Debug("source %s", src) lsc.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + }) } return result, nil diff --git a/internal/pkg/helm/local_stored_collector_test.go b/internal/pkg/helm/local_stored_collector_test.go index 5d813432d..ed7eca3f1 100644 --- a/internal/pkg/helm/local_stored_collector_test.go +++ b/internal/pkg/helm/local_stored_collector_test.go @@ -709,8 +709,8 @@ func TestHelmImageCollector(t *testing.T) { } if len(testCase.expectedResult) > 0 { - assert.NotEmpty(t, imgs) - assert.ElementsMatch(t, testCase.expectedResult, imgs) + assert.NotEmpty(t, imgs.AllImages) + assert.ElementsMatch(t, testCase.expectedResult, imgs.AllImages) } }) } @@ -873,7 +873,7 @@ func TestHelmImageCollectorVPrefixDiskToMirror(t *testing.T) { imgs, err := helmCollector.HelmImageCollector(ctx) assert.NoError(t, err) - assert.ElementsMatch(t, tc.expectedImages, imgs) + assert.ElementsMatch(t, tc.expectedImages, imgs.AllImages) }) } } diff --git a/internal/pkg/mirror/mirror.go b/internal/pkg/mirror/mirror.go index 9f1d04471..329e5f815 100644 --- a/internal/pkg/mirror/mirror.go +++ b/internal/pkg/mirror/mirror.go @@ -135,20 +135,9 @@ func (o *Mirror) copy(ctx context.Context, src, dest string, opts *CopyOptions) } } - imageListSelection := copy.CopySystemImage - if len(opts.MultiArch) > 0 && opts.All { - return fmt.Errorf("cannot use --all and --multi-arch flags together") - } - - if len(opts.MultiArch) > 0 { - imageListSelection, err = parseMultiArch(opts.MultiArch) - if err != nil { - return err - } - } - - if opts.All { - imageListSelection = copy.CopyAllImages + imageListSelection, instancePlatforms, err := determinePlatformSelection(opts) + if err != nil { + return err } if len(opts.EncryptionKeys) > 0 && len(opts.DecryptionKeys) > 0 { @@ -193,6 +182,7 @@ func (o *Mirror) copy(ctx context.Context, src, dest string, opts *CopyOptions) DestinationCtx: destinationCtx, ForceManifestMIMEType: manifestType, ImageListSelection: imageListSelection, + InstancePlatforms: instancePlatforms, PreserveDigests: opts.PreserveDigests, MaxParallelDownloads: opts.ParallelLayerImages, } @@ -323,22 +313,69 @@ func (o *Mirror) delete(ctx context.Context, image string, opts *CopyOptions) er }, opts.RetryOpts) } -// parseMultiArch -func parseMultiArch(multiArch string) (copy.ImageListSelection, error) { +// determinePlatformSelection determines the image list selection and platform filters +// based on the provided copy options. It prioritizes InstancePlatforms (from ImageSetConfig) +// over the MultiArch flag. +func determinePlatformSelection(opts *CopyOptions) (copy.ImageListSelection, []copy.InstancePlatformFilter, error) { + // Priority 1: Use InstancePlatforms if provided (set by batch worker from ImageSetConfig) + if len(opts.InstancePlatforms) > 0 { + platformsStr := strings.Join(opts.InstancePlatforms, ",") + return parseMultiArch(platformsStr) + } + + // Priority 2: Fall back to MultiArch flag (for CLI direct usage or default "all") + if len(opts.MultiArch) > 0 && opts.All { + return copy.CopySystemImage, nil, fmt.Errorf("cannot use --all and --multi-arch flags together") + } + + imageListSelection := copy.CopySystemImage + var instancePlatforms []copy.InstancePlatformFilter + var err error + + if len(opts.MultiArch) > 0 { + imageListSelection, instancePlatforms, err = parseMultiArch(opts.MultiArch) + if err != nil { + return copy.CopySystemImage, nil, err + } + } + + if opts.All { + imageListSelection = copy.CopyAllImages + } + + return imageListSelection, instancePlatforms, nil +} + +// parseMultiArch parses the multi-arch option and returns the image list selection +// and optional platform filters for specific OS/Architecture combinations. +// Supports legacy options ('system', 'all', 'index-only') and new platform +// specifications (e.g., 'linux/amd64,linux/arm64'). +func parseMultiArch(multiArch string) (copy.ImageListSelection, []copy.InstancePlatformFilter, error) { switch multiArch { case "system": - return copy.CopySystemImage, nil + return copy.CopySystemImage, nil, nil case "all": - return copy.CopyAllImages, nil + return copy.CopyAllImages, nil, nil // There is no CopyNoImages value in copy.ImageListSelection, but because we // don't provide an option to select a set of images to copy, we can use // CopySpecificImages. case "index-only": - return copy.CopySpecificImages, nil - // We don't expose CopySpecificImages other than index-only above, because - // we currently don't provide an option to choose the images to copy. That - // could be added in the future. + return copy.CopySpecificImages, nil, nil default: - return copy.CopySystemImage, fmt.Errorf("unknown multi-arch option %q. Choose one of the supported options: 'system', 'all', or 'index-only'", multiArch) + // Try to parse as comma-separated platform list (e.g., "linux/amd64,linux/arm64") + // Parse comma-separated platform list + var platforms []copy.InstancePlatformFilter + for _, platform := range strings.Split(multiArch, ",") { + platform = strings.TrimSpace(platform) + parts := strings.Split(platform, "/") + if len(parts) != 2 { + return copy.CopySystemImage, nil, fmt.Errorf("unknown multi-arch option %q. Choose one of the supported options: 'system', 'all', 'index-only', or a comma-separated platform list like 'linux/amd64,linux/arm64'", multiArch) + } + platforms = append(platforms, copy.InstancePlatformFilter{ + OS: parts[0], + Architecture: parts[1], + }) + } + return copy.CopySpecificImages, platforms, nil } } diff --git a/internal/pkg/mirror/mirror_test.go b/internal/pkg/mirror/mirror_test.go index 58810cf62..1b4cc1f1e 100644 --- a/internal/pkg/mirror/mirror_test.go +++ b/internal/pkg/mirror/mirror_test.go @@ -67,7 +67,7 @@ func TestMirrorCopy(t *testing.T) { opts.MultiArch = "other" t.Run("Testing Mirror : copy should fail", func(t *testing.T) { err := m.Run(context.Background(), consts.DockerProtocol+"localhost.localdomain:5000/tes", "oci:test", "copy", &opts) - assert.Equal(t, "unknown multi-arch option \"other\". Choose one of the supported options: 'system', 'all', or 'index-only'", err.Error()) + assert.Equal(t, "unknown multi-arch option \"other\". Choose one of the supported options: 'system', 'all', 'index-only', or a comma-separated platform list like 'linux/amd64,linux/arm64'", err.Error()) }) opts.All = true @@ -225,17 +225,35 @@ func TestMirrorDelete(t *testing.T) { // TestMirrorParseMultiArch func TestMirrorParseMultiArch(t *testing.T) { - res, _ := parseMultiArch("system") + res, platforms, err := parseMultiArch("system") + assert.NoError(t, err) assert.Equal(t, copy.ImageListSelection(0), res) + assert.Nil(t, platforms) - res, _ = parseMultiArch("all") + res, platforms, err = parseMultiArch("all") + assert.NoError(t, err) assert.Equal(t, copy.ImageListSelection(1), res) + assert.Nil(t, platforms) - res, _ = parseMultiArch("index-only") + res, platforms, err = parseMultiArch("index-only") + assert.NoError(t, err) assert.Equal(t, copy.ImageListSelection(2), res) - - _, err := parseMultiArch("other") - assert.Equal(t, "unknown multi-arch option \"other\". Choose one of the supported options: 'system', 'all', or 'index-only'", err.Error()) + assert.Nil(t, platforms) + + // Test platform specification + res, platforms, err = parseMultiArch("linux/amd64,linux/arm64") + assert.NoError(t, err) + assert.Equal(t, copy.ImageListSelection(2), res) // CopySpecificImages + assert.Len(t, platforms, 2) + assert.Equal(t, "linux", platforms[0].OS) + assert.Equal(t, "amd64", platforms[0].Architecture) + assert.Equal(t, "linux", platforms[1].OS) + assert.Equal(t, "arm64", platforms[1].Architecture) + + // Test invalid option + _, _, err = parseMultiArch("other") + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown multi-arch option") } type ( diff --git a/internal/pkg/mirror/options.go b/internal/pkg/mirror/options.go index 331b7598d..ad2f2d97d 100644 --- a/internal/pkg/mirror/options.go +++ b/internal/pkg/mirror/options.go @@ -79,6 +79,7 @@ type CopyOptions struct { Format string // Force conversion of the image to a specified format All bool // Copy all of the images if the source is a list MultiArch string // How to handle multi architecture images + InstancePlatforms []string // Platform specifications (e.g., "linux/amd64", "linux/arm64") for multi-arch images PreserveDigests bool // Preserve digests during copy EncryptLayer []int // The list of layers to encrypt EncryptionKeys []string // Keys needed to encrypt the image diff --git a/internal/pkg/operator/filtered_collector.go b/internal/pkg/operator/filtered_collector.go index 5ad831303..ead86e6a8 100644 --- a/internal/pkg/operator/filtered_collector.go +++ b/internal/pkg/operator/filtered_collector.go @@ -39,6 +39,7 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. relatedImages := make(map[string][]v2alpha1.RelatedImage) collectorSchema := v2alpha1.CollectorSchema{ CatalogToFBCMap: make(map[string]v2alpha1.CatalogFilterResult), + PlatformFilters: make(map[string][]string), } copyImageSchemaMap := &v2alpha1.CopyImageSchemaMap{ OperatorsByImage: make(map[string]map[string]struct{}), @@ -70,13 +71,34 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. continue } - result, err := o.collectOperator(ctx, op, relatedImages, copyImageSchemaMap) + // Use a fresh local map per catalog so we can read exactly which images this + // catalog contributed — needed to populate platform filters correctly when the + // same bundle appears in multiple catalogs with different platform requirements. + localRelatedImages := make(map[string][]v2alpha1.RelatedImage) + result, err := o.collectOperator(ctx, op, localRelatedImages, copyImageSchemaMap) if err != nil { spinner.Abort(true) spinner.Wait() allErrs = append(allErrs, fmt.Errorf("collect catalog %q: %w", op.Catalog, err)) continue } + + // Append this catalog's platform filters into PlatformFilters using localRelatedImages. + // The same bundle appearing in two catalogs with different platforms gets both appended. + if platforms := v2alpha1.ConvertPlatformsToStringSlice(op.Platforms); len(platforms) > 0 { + for _, imgs := range localRelatedImages { + for _, img := range imgs { + if ref, parseErr := image.ParseRef(img.Image); parseErr == nil { + origin := ref.ReferenceWithTransport + collectorSchema.PlatformFilters[origin] = append(collectorSchema.PlatformFilters[origin], platforms...) + } + } + } + } + + // Merge this catalog's images into the global map. + maps.Copy(relatedImages, localRelatedImages) + // OCPBUGS-81712: In M2D/M2M modes, op.Catalog is already pinned to digest by executor.go // CLID-513: For OCI paths with digest, use a consistent key format (without digest) // This matches how catalogImage is constructed in collectOperator diff --git a/internal/pkg/release/cincinnati.go b/internal/pkg/release/cincinnati.go index 0b29eef16..71f4f0960 100644 --- a/internal/pkg/release/cincinnati.go +++ b/internal/pkg/release/cincinnati.go @@ -159,109 +159,29 @@ func (o *CincinnatiSchema) GetReleaseReferenceImages(ctx context.Context) ([]v2a filterCopy := o.Config.Mirror.Platform.DeepCopy() - for _, arch := range filterCopy.Architectures { - cincinnatiParams.Arch = arch + if len(filterCopy.Platforms) > 0 { + // New Platforms field: always use the multi-arch payload. It is the only manifest + // list, enabling sparse manifest platform filtering. Requires the target registry + // to support sparse manifest lists (e.g. validation.manifests.indexes.platforms: none). + o.Log.Warn("Platform filtering with the Platforms field requires a registry with manifest list " + + "blob check disabled (e.g. validation.manifests.indexes.platforms: none). " + + "Ensure your target registry supports sparse manifest lists.") + cincinnatiParams.Arch = "multi" o.CincinnatiParams = cincinnatiParams - versionsByChannel := make(map[string]v2alpha1.ReleaseChannel, len(filterCopy.Channels)) - for _, ch := range filterCopy.Channels { - var err error - switch ch.Type { - case v2alpha1.TypeOCP: - err = o.NewOCPClient() - if err != nil { - errs = append(errs, err) - } - case v2alpha1.TypeOKD: - err = o.NewOKDClient() - if err != nil { - errs = append(errs, err) - } - default: - errs = append(errs, fmt.Errorf("invalid platform type %v", ch.Type)) - continue - } - if err != nil { - errs = append(errs, err) - continue - } - - // CLID-135 - // detect and log as early as possible - if len(ch.MaxVersion) > 0 && len(ch.MinVersion) > 0 { - max := semver.MustParse(ch.MaxVersion) - min := semver.MustParse(ch.MinVersion) - if strings.Contains(ch.Name, "eus") && ((max.Minor - min.Minor) >= 2) && !flagReport { - msg := "Extended Update Support (EUS) channel detected with minor version range >= 2\n" + - "\t\t\t\tPlease refer to the web console https://access.redhat.com/labs/ocpupgradegraph/update_path\n" + - "\t\t\t\tTo correctly determine the upgrade path for EUS releases" - flagReport = true - o.Log.Warn(msg) - } - } - - if len(ch.MaxVersion) == 0 || len(ch.MinVersion) == 0 { - // Find channel maximum value and only set the minimum as well if heads-only is true - if len(ch.MaxVersion) == 0 { - latest, err := GetChannelMinOrMax(ctx, *o, ch.Name, false) - if err != nil { - errs = append(errs, err) - continue - } - - // Update version to release channel - ch.MaxVersion = latest.String() - o.Log.Debug("detected minimum version as %s", ch.MaxVersion) - if len(ch.MinVersion) == 0 && ch.IsHeadsOnly() { - min := latest.String() - ch.MinVersion = min - o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) - } - } - - // Find channel minimum if full is true or just the minimum is not set - // in the config - if len(ch.MinVersion) == 0 { - first, err := GetChannelMinOrMax(ctx, *o, ch.Name, true) - if err != nil { - errs = append(errs, err) - continue - } - ch.MinVersion = first.String() - o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) - } - versionsByChannel[ch.Name] = ch - } else { - // Range is set. Ensure full is true so this - // is skipped when processing release metadata. - o.Log.Debug("processing minimum version %s and maximum version %s", ch.MinVersion, ch.MaxVersion) - ch.Full = true - versionsByChannel[ch.Name] = ch - } - - downloads, err := getChannelDownloads(ctx, *o, nil, ch) - if err != nil { - errs = append(errs, err) - continue - } - allImages = append(allImages, downloads...) - } - - // Update cfg release channels with maximum and minimum versions - // if applicable - for i, ch := range filterCopy.Channels { - ch, found := versionsByChannel[ch.Name] - if found { - filterCopy.Channels[i] = ch - } - } - - if len(filterCopy.Channels) > 1 { - newDownloads, err := getCrossChannelDownloads(ctx, *o, filterCopy.Channels) - if err != nil { - errs = append(errs, fmt.Errorf("[GetReleaseReferenceImages] error calculating cross channel upgrades: %w", err)) - continue - } - allImages = append(allImages, newDownloads...) + imgs, channelErrs := o.collectChannelImages(ctx, filterCopy.Channels, &flagReport) + allImages = append(allImages, imgs...) + errs = append(errs, channelErrs...) + } else { + // Deprecated Architectures field: original per-arch loop. + // Each arch results in a separate Cincinnati call and a separate release payload. + o.Log.Warn("platform.architectures is deprecated; use platform.platforms instead for sparse manifest platform filtering") + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility + for _, arch := range filterCopy.Architectures { + cincinnatiParams.Arch = arch + o.CincinnatiParams = cincinnatiParams + imgs, channelErrs := o.collectChannelImages(ctx, filterCopy.Channels, &flagReport) + allImages = append(allImages, imgs...) + errs = append(errs, channelErrs...) } } @@ -285,6 +205,119 @@ func (o *CincinnatiSchema) GetReleaseReferenceImages(ctx context.Context) ([]v2a return imgs, nil } +// collectChannelImages queries the Cincinnati API for all configured channels with +// the arch already set on o.CincinnatiParams, returning collected images and errors. +// The channels slice is updated in place with resolved min/max versions. +// flagReport is shared across calls to ensure the EUS warning is emitted only once. +func (o *CincinnatiSchema) collectChannelImages(ctx context.Context, channels []v2alpha1.ReleaseChannel, flagReport *bool) ([]v2alpha1.CopyImageSchema, []error) { + var allImages []v2alpha1.CopyImageSchema + var errs []error + + versionsByChannel := make(map[string]v2alpha1.ReleaseChannel, len(channels)) + for _, ch := range channels { + if err := o.initCincinnatiClient(ch); err != nil { + errs = append(errs, err) + continue + } + o.warnEUSChannel(ch, flagReport) + + resolved, err := o.resolveChannelVersions(ctx, ch) + if err != nil { + errs = append(errs, err) + continue + } + versionsByChannel[resolved.Name] = resolved + + downloads, err := getChannelDownloads(ctx, *o, nil, resolved) + if err != nil { + errs = append(errs, err) + continue + } + allImages = append(allImages, downloads...) + } + + // Update channels with resolved min/max versions + for i, ch := range channels { + if updated, found := versionsByChannel[ch.Name]; found { + channels[i] = updated + } + } + + if len(channels) > 1 { + newDownloads, err := getCrossChannelDownloads(ctx, *o, channels) + if err != nil { + errs = append(errs, fmt.Errorf("[GetReleaseReferenceImages] error calculating cross channel upgrades: %w", err)) + } else { + allImages = append(allImages, newDownloads...) + } + } + + return allImages, errs +} + +// initCincinnatiClient sets up the OCP or OKD Cincinnati client for the given channel type. +func (o *CincinnatiSchema) initCincinnatiClient(ch v2alpha1.ReleaseChannel) error { + switch ch.Type { + case v2alpha1.TypeOCP: + return o.NewOCPClient() + case v2alpha1.TypeOKD: + return o.NewOKDClient() + default: + return fmt.Errorf("invalid platform type %v", ch.Type) + } +} + +// warnEUSChannel emits a one-time warning when an EUS channel spans two or more minor versions. +func (o *CincinnatiSchema) warnEUSChannel(ch v2alpha1.ReleaseChannel, flagReport *bool) { + if len(ch.MaxVersion) == 0 || len(ch.MinVersion) == 0 || *flagReport { + return + } + // CLID-135: detect and log as early as possible + max := semver.MustParse(ch.MaxVersion) + min := semver.MustParse(ch.MinVersion) + if strings.Contains(ch.Name, "eus") && (max.Minor-min.Minor) >= 2 { + o.Log.Warn("Extended Update Support (EUS) channel detected with minor version range >= 2\n" + + "\t\t\t\tPlease refer to the web console https://access.redhat.com/labs/ocpupgradegraph/update_path\n" + + "\t\t\t\tTo correctly determine the upgrade path for EUS releases") + *flagReport = true + } +} + +// resolveChannelVersions detects and fills in missing min/max version bounds for a channel, +// or marks the channel as full when both are explicitly provided. +func (o *CincinnatiSchema) resolveChannelVersions(ctx context.Context, ch v2alpha1.ReleaseChannel) (v2alpha1.ReleaseChannel, error) { + if len(ch.MaxVersion) > 0 && len(ch.MinVersion) > 0 { + // Range is set. Ensure full is true so this is skipped when processing release metadata. + o.Log.Debug("processing minimum version %s and maximum version %s", ch.MinVersion, ch.MaxVersion) + ch.Full = true + return ch, nil + } + + if len(ch.MaxVersion) == 0 { + latest, err := GetChannelMinOrMax(ctx, *o, ch.Name, false) + if err != nil { + return ch, err + } + ch.MaxVersion = latest.String() + o.Log.Debug("detected maximum version as %s", ch.MaxVersion) + if len(ch.MinVersion) == 0 && ch.IsHeadsOnly() { + ch.MinVersion = latest.String() + o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) + } + } + + if len(ch.MinVersion) == 0 { + first, err := GetChannelMinOrMax(ctx, *o, ch.Name, true) + if err != nil { + return ch, err + } + ch.MinVersion = first.String() + o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) + } + + return ch, nil +} + // getDownloads will prepare the downloads map for mirroring func getChannelDownloads(ctx context.Context, cs CincinnatiSchema, lastChannels []v2alpha1.ReleaseChannel, channel v2alpha1.ReleaseChannel) ([]v2alpha1.CopyImageSchema, error) { var allImages []v2alpha1.CopyImageSchema diff --git a/internal/pkg/release/interface.go b/internal/pkg/release/interface.go index 5fe7b17dd..3ca4c4b48 100644 --- a/internal/pkg/release/interface.go +++ b/internal/pkg/release/interface.go @@ -7,7 +7,7 @@ import ( ) type CollectorInterface interface { - ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) + ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) // Returns the graphImage if generated, especially with the reference to the image // on the destination repository // Returns an error if the graph image was not generated (graph : false) diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 4cd24261c..6582a9f47 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -50,9 +50,21 @@ func (o LocalStorageCollector) destinationRegistry() string { return o.destReg } -func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +// getPlatformFilters returns the platform filter strings for the new Platforms field. +// Returns nil when Platforms is not set — the Architectures path uses Cincinnati +// directly and does not need InstancePlatforms in the batch worker. +func (o LocalStorageCollector) getPlatformFilters() []string { + return v2alpha1.ConvertPlatformsToStringSlice(o.Config.Mirror.Platform.Platforms) +} + +func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { // we just care for 1 platform release, in order to read release images o.Opts.MultiArch = "system" + if len(o.Config.Mirror.Platform.Platforms) > 0 { + // New Platforms field: download all arch payloads from the multi-arch release so that + // ensureReleaseInOCIFormat can extract image-references with manifest-list component digests. + o.Opts.MultiArch = "all" + } o.Log.Debug(collectorPrefix+"setting copy option o.Opts.MultiArch=%s when collecting releases image", o.Opts.MultiArch) var err error @@ -68,24 +80,29 @@ func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) ([]v2 err = fmt.Errorf("release collector: invalid mirror mode %s", o.Opts.Mode) } if err != nil { - return []v2alpha1.CopyImageSchema{}, err + return v2alpha1.CollectorSchema{}, err } // OCPBUGS-43275: deduplicating - slices.SortFunc(allImages, func(a, b v2alpha1.CopyImageSchema) int { - cmp := strings.Compare(a.Origin, b.Origin) - if cmp == 0 { - cmp = strings.Compare(a.Source, b.Source) - } - if cmp == 0 { // this comparison is important because the same digest can be used - // several times in image-references for different components - cmp = strings.Compare(a.Destination, b.Destination) - } - return cmp - }) + slices.SortFunc(allImages, compareByOriginSourceDest) allImages = slices.Compact(allImages) - return allImages, nil + // TODO probably does not need to loop all the images, it would be possible to check the image type, if it is from a release, apply always the same os/arch saved somewhere else. + // TBD with team - Keeping in this way the advantages is that we create a standard and don't treat releases in a different way + // The disadvantage is performance + cs := v2alpha1.CollectorSchema{AllImages: allImages} + if platforms := o.getPlatformFilters(); len(platforms) > 0 { + cs.PlatformFilters = make(map[string][]string, len(allImages)) + for _, img := range allImages { + // Graph images are excluded: in the non-enclave case they are already built for + // the requested architectures by CreateGraphImage; in the enclave case they are + // pre-built external artifacts that must be copied whole. + if img.Type != v2alpha1.TypeCincinnatiGraph { + cs.PlatformFilters[img.Origin] = platforms + } + } + } + return cs, nil } // collects release images from a mirror. @@ -115,7 +132,11 @@ func (o *LocalStorageCollector) collectImageFromMirror(ctx context.Context) ([]v } // add the release image itself - allRelatedImages = append(allRelatedImages, v2alpha1.RelatedImage{Image: value.Source, Name: value.Source, Type: v2alpha1.TypeOCPRelease}) + allRelatedImages = append(allRelatedImages, v2alpha1.RelatedImage{ + Image: value.Source, + Name: value.Source, + Type: v2alpha1.TypeOCPRelease, + }) tmpAllImages, err := o.prepareM2DCopyBatch(allRelatedImages, releaseTag) if err != nil { logCollectionError(o.Log, spinner, o.Opts.Global.IsTerminal, value.Source, err) @@ -200,12 +221,13 @@ func (o *LocalStorageCollector) ensureReleaseInOCIFormat(ctx context.Context, re optsCopy := o.Opts optsCopy.Stdout = io.Discard + // Converting docker→OCI changes the digest, which invalidates existing signatures. + // This copy is for metadata extraction only, so signatures are not needed. + optsCopy.RemoveSignatures = true src := consts.DockerProtocol + release.Source dest := consts.OciProtocolTrimmed + dir - optsCopy.RemoveSignatures = true - if err := o.Mirror.Run(ctx, src, dest, "copy", &optsCopy); err != nil { return fmt.Errorf("copy release index image: %w", err) } @@ -355,7 +377,12 @@ func (o LocalStorageCollector) prepareM2DCopyBatch(images []v2alpha1.RelatedImag o.Log.Debug("source %s", src) o.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + }) } return result, nil } @@ -383,7 +410,12 @@ func (o LocalStorageCollector) prepareD2MCopyBatch(images []v2alpha1.RelatedImag o.Log.Debug("source %s", src) o.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + }) } return result, nil @@ -417,7 +449,11 @@ func (o LocalStorageCollector) identifyReleases(ctx context.Context) ([]v2alpha1 releasePath = strings.TrimPrefix(releasePath, consts.OciProtocolTrimmed) releaseHoldPath := strings.Replace(releasePath, releaseImageDir, releaseImageExtractDir, 1) releaseFolders = append(releaseFolders, releaseHoldPath) - releaseImages = append(releaseImages, v2alpha1.RelatedImage{Name: copy.Source, Image: copy.Source, Type: v2alpha1.TypeOCPRelease}) + releaseImages = append(releaseImages, v2alpha1.RelatedImage{ + Name: copy.Source, + Image: copy.Source, + Type: v2alpha1.TypeOCPRelease, + }) } return releaseImages, releaseFolders, nil } @@ -564,6 +600,8 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C } } else { + // TODO: pass o.getPlatformFilters() to CreateGraphImage so the graph image is built + // only for the requested architectures. Tracked as a separate PR. graphImgRef, err := o.CreateGraphImage(ctx, graphURL) if err != nil { return v2alpha1.CopyImageSchema{}, err @@ -581,6 +619,19 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C } } +// compareByOriginSourceDest sorts CopyImageSchema entries deterministically. +// The same digest can appear several times in image-references for different +// components, so all three fields are compared to produce a stable order. +func compareByOriginSourceDest(a, b v2alpha1.CopyImageSchema) int { + if cmp := strings.Compare(a.Origin, b.Origin); cmp != 0 { + return cmp + } + if cmp := strings.Compare(a.Source, b.Source); cmp != 0 { + return cmp + } + return strings.Compare(a.Destination, b.Destination) +} + func preparePathComponents(imgSpec image.ImageSpec, imgType v2alpha1.ImageType, imgName string) string { pathComponents := "" switch { diff --git a/internal/pkg/release/local_stored_collector_test.go b/internal/pkg/release/local_stored_collector_test.go index f29c2ed6d..351399211 100644 --- a/internal/pkg/release/local_stored_collector_test.go +++ b/internal/pkg/release/local_stored_collector_test.go @@ -111,7 +111,7 @@ func TestReleaseLocalStoredCollector(t *testing.T) { Type: v2alpha1.TypeCincinnatiGraph, }, } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) log.Debug("completed test related images %v ", res) }) @@ -137,10 +137,10 @@ func TestReleaseLocalStoredCollector(t *testing.T) { if err != nil { t.Fatalf("should not fail: %v", err) } - if len(res) == 0 { + if len(res.AllImages) == 0 { t.Fatalf("should contain at least 1 image") } - if !strings.Contains(res[0].Source, ex.LocalStorageFQDN) { + if !strings.Contains(res.AllImages[0].Source, ex.LocalStorageFQDN) { t.Fatalf("source images should be from local storage") } // must contain 4 release component images @@ -191,7 +191,7 @@ func TestReleaseLocalStoredCollector(t *testing.T) { Type: v2alpha1.TypeCincinnatiGraph, }, } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) log.Debug("completed test related images %v ", res) }) @@ -218,10 +218,10 @@ func TestReleaseLocalStoredCollector(t *testing.T) { if err != nil { t.Fatalf("should not fail: %v", err) } - if len(res) == 0 { + if len(res.AllImages) == 0 { t.Fatalf("should contain at least 1 image") } - if !strings.Contains(res[0].Source, ex.LocalStorageFQDN) { + if !strings.Contains(res.AllImages[0].Source, ex.LocalStorageFQDN) { t.Fatalf("source images should be from local storage") } log.Debug("completed test related images %v ", res) diff --git a/tests/integration/helpers_test.go b/tests/integration/helpers_test.go index 4b7609e6f..bb2e5fc0c 100644 --- a/tests/integration/helpers_test.go +++ b/tests/integration/helpers_test.go @@ -512,7 +512,7 @@ func setupWorkDir() string { err = os.MkdirAll(sigDir, 0o755) Expect(err).NotTo(HaveOccurred()) - // Find the single signature file in keys/ (any file that isn't the public key) + // Copy all release signature files from keys/ (any file that isn't the public key or cosign key) entries, err := os.ReadDir(keysDir) Expect(err).NotTo(HaveOccurred()) @@ -522,10 +522,12 @@ func setupWorkDir() string { sigFiles = append(sigFiles, entry.Name()) } } - Expect(sigFiles).To(HaveLen(1), "expected exactly one signature file in keys/, found: %v", sigFiles) + Expect(sigFiles).NotTo(BeEmpty(), "no signature files found in keys/") - err = copyFile(filepath.Join(keysDir, sigFiles[0]), filepath.Join(sigDir, sigFiles[0])) - Expect(err).NotTo(HaveOccurred()) + for _, sigFile := range sigFiles { + err = copyFile(filepath.Join(keysDir, sigFile), filepath.Join(sigDir, sigFile)) + Expect(err).NotTo(HaveOccurred()) + } return workDir } diff --git a/tests/integration/image-builders/release/Makefile b/tests/integration/image-builders/release/Makefile index 3460fb238..804bb6457 100755 --- a/tests/integration/image-builders/release/Makefile +++ b/tests/integration/image-builders/release/Makefile @@ -26,3 +26,7 @@ sign: COSIGN_PASSWORD="" cosign sign --key $(COSIGN_KEY) --tlog-upload=false quay.io/oc-mirror/release/test-release-index:v0.0.1 .PHONY: sign +sign-multi: + COSIGN_PASSWORD="" cosign sign --recursive --key $(COSIGN_KEY) --tlog-upload=false quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 +.PHONY: sign-multi + diff --git a/tests/integration/image-builders/release/artifacts/config/2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2 b/tests/integration/image-builders/release/artifacts/config/2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2 new file mode 100644 index 000000000..a4413707d --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2 @@ -0,0 +1,48 @@ +{ + "created": "2023-01-24T02:27:38Z", + "architecture": "arm64", + "os": "linux", + "config": { + "User": "0", + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "container=oci", + "__doozer=merge", + "BUILD_RELEASE=202212061458.p0.g7e8a010.assembly.stream", + "BUILD_VERSION=v4.12.0", + "OS_GIT_MAJOR=4", + "OS_GIT_MINOR=12", + "OS_GIT_PATCH=0", + "OS_GIT_TREE_STATE=clean", + "OS_GIT_VERSION=4.12.0-202212061458.p0.g7e8a010.assembly.stream-7e8a010", + "SOURCE_GIT_TREE_STATE=clean", + "OS_GIT_COMMIT=7e8a010", + "SOURCE_DATE_EPOCH=1666997438", + "SOURCE_GIT_COMMIT=7e8a0105eb7369f3f92ad7b2581a2efffab5b28e", + "SOURCE_GIT_TAG=7e8a010", + "SOURCE_GIT_URL=https://github.com/oc-mirror-test/integration", + "GODEBUG=x509ignoreCN=0,madvdontneed=1", + "OPENSHIFT_CI=true", + "BUILD_LOGLEVEL=0", + "OPENSHIFT_BUILD_NAME=test-release-image", + "OPENSHIFT_BUILD_NAMESPACE=ci-op-1c0qqwdy" + ], + "Entrypoint": [ + "/usr/bin/simple-release" + ], + "Labels": { + "io.openshift.release": "4.18.0-0.ocp-2025-03-24-022732", + "io.openshift.release.base-image-digest": "sha256:0ed55ee636f5703b212337ff8a82833c5dfc4553069dc3f6cd52564cc0adcd08" + } + }, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2025-03-24T02:27:38Z", + "comment": "Release image for testing oc-mirror v2" + } + ] +} diff --git a/tests/integration/image-builders/release/artifacts/config/87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956 b/tests/integration/image-builders/release/artifacts/config/87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956 new file mode 100644 index 000000000..ec10d66f8 --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956 @@ -0,0 +1,48 @@ +{ + "created": "2023-01-24T02:27:38Z", + "architecture": "ppc64le", + "os": "linux", + "config": { + "User": "0", + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "container=oci", + "__doozer=merge", + "BUILD_RELEASE=202212061458.p0.g7e8a010.assembly.stream", + "BUILD_VERSION=v4.12.0", + "OS_GIT_MAJOR=4", + "OS_GIT_MINOR=12", + "OS_GIT_PATCH=0", + "OS_GIT_TREE_STATE=clean", + "OS_GIT_VERSION=4.12.0-202212061458.p0.g7e8a010.assembly.stream-7e8a010", + "SOURCE_GIT_TREE_STATE=clean", + "OS_GIT_COMMIT=7e8a010", + "SOURCE_DATE_EPOCH=1666997438", + "SOURCE_GIT_COMMIT=7e8a0105eb7369f3f92ad7b2581a2efffab5b28e", + "SOURCE_GIT_TAG=7e8a010", + "SOURCE_GIT_URL=https://github.com/oc-mirror-test/integration", + "GODEBUG=x509ignoreCN=0,madvdontneed=1", + "OPENSHIFT_CI=true", + "BUILD_LOGLEVEL=0", + "OPENSHIFT_BUILD_NAME=test-release-image", + "OPENSHIFT_BUILD_NAMESPACE=ci-op-1c0qqwdy" + ], + "Entrypoint": [ + "/usr/bin/simple-release" + ], + "Labels": { + "io.openshift.release": "4.18.0-0.ocp-2025-03-24-022732", + "io.openshift.release.base-image-digest": "sha256:0ed55ee636f5703b212337ff8a82833c5dfc4553069dc3f6cd52564cc0adcd08" + } + }, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2025-03-24T02:27:38Z", + "comment": "Release image for testing oc-mirror v2" + } + ] +} diff --git a/tests/integration/image-builders/release/artifacts/config/config-multi.json b/tests/integration/image-builders/release/artifacts/config/config-multi.json new file mode 100644 index 000000000..cd944b386 --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/config-multi.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:CONFIG_DIGEST", + "size": CONFIG_SIZE + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:LAYER_DIGEST", + "size": LAYER_SIZE + } + ] +} diff --git a/tests/integration/image-builders/release/artifacts/config/dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb b/tests/integration/image-builders/release/artifacts/config/dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb new file mode 100644 index 000000000..20f912481 --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb @@ -0,0 +1,48 @@ +{ + "created": "2023-01-24T02:27:38Z", + "architecture": "s390x", + "os": "linux", + "config": { + "User": "0", + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "container=oci", + "__doozer=merge", + "BUILD_RELEASE=202212061458.p0.g7e8a010.assembly.stream", + "BUILD_VERSION=v4.12.0", + "OS_GIT_MAJOR=4", + "OS_GIT_MINOR=12", + "OS_GIT_PATCH=0", + "OS_GIT_TREE_STATE=clean", + "OS_GIT_VERSION=4.12.0-202212061458.p0.g7e8a010.assembly.stream-7e8a010", + "SOURCE_GIT_TREE_STATE=clean", + "OS_GIT_COMMIT=7e8a010", + "SOURCE_DATE_EPOCH=1666997438", + "SOURCE_GIT_COMMIT=7e8a0105eb7369f3f92ad7b2581a2efffab5b28e", + "SOURCE_GIT_TAG=7e8a010", + "SOURCE_GIT_URL=https://github.com/oc-mirror-test/integration", + "GODEBUG=x509ignoreCN=0,madvdontneed=1", + "OPENSHIFT_CI=true", + "BUILD_LOGLEVEL=0", + "OPENSHIFT_BUILD_NAME=test-release-image", + "OPENSHIFT_BUILD_NAMESPACE=ci-op-1c0qqwdy" + ], + "Entrypoint": [ + "/usr/bin/simple-release" + ], + "Labels": { + "io.openshift.release": "4.18.0-0.ocp-2025-03-24-022732", + "io.openshift.release.base-image-digest": "sha256:0ed55ee636f5703b212337ff8a82833c5dfc4553069dc3f6cd52564cc0adcd08" + } + }, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2025-03-24T02:27:38Z", + "comment": "Release image for testing oc-mirror v2" + } + ] +} diff --git a/tests/integration/image-builders/release/create-release-multi.sh b/tests/integration/image-builders/release/create-release-multi.sh new file mode 100755 index 000000000..0435b839e --- /dev/null +++ b/tests/integration/image-builders/release/create-release-multi.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Builds a multi-arch fake OCP release payload OCI layout and pushes it as +# quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 +# +# Prerequisites: skopeo, jq, sha256sum, stat +# Run from the image-builders/release/ directory. +# +# After a successful run: +# make sign-multi (cosign signature) +# ./generate-release-signature.sh (GPG atomic container signature) +# Commit: release-payload-multi/index.json and the new GPG signature in ../../testdata/keys/ + +set -ex + +REPO="quay.io/oc-mirror/release/test-release-index-multi" +TAG="v0.0.1" + +# Per-arch config blob filenames (sha256 of content, pre-committed) +declare -A ARCH_CONFIG_BLOB=( + ["amd64"]="8366d414d18526e99f4781ed84a93b5b96ebe464d223e587cf7f705829b27a12" + ["arm64"]="2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2" + ["s390x"]="dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb" + ["ppc64le"]="87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956" +) + +rm -rf release-payload-multi/blobs/sha256 +mkdir -p release-payload-multi/blobs/sha256 artifacts/staging + +# Step 1: Build the shared release-manifests layer blob (same for all arches) +cd artifacts/staging +tar -czvf tmp_layer.tar.gz ../release-manifests/ +LAYER_DIGEST=$(sha256sum tmp_layer.tar.gz | cut -d ' ' -f 1) +LAYER_SIZE=$(stat --printf="%s" tmp_layer.tar.gz) +cp tmp_layer.tar.gz ../../release-payload-multi/blobs/sha256/${LAYER_DIGEST} +cd ../../ + +# Step 2: For each arch, build an OCI image manifest referencing the shared layer +MANIFESTS_JSON="" + +for ARCH in amd64 arm64 s390x ppc64le; do + CONFIG_BLOB_NAME="${ARCH_CONFIG_BLOB[$ARCH]}" + CONFIG_BLOB_PATH="artifacts/config/${CONFIG_BLOB_NAME}" + CONFIG_SIZE=$(stat --printf="%s" "${CONFIG_BLOB_PATH}") + + # Copy config blob to payload blobs directory + cp "${CONFIG_BLOB_PATH}" release-payload-multi/blobs/sha256/${CONFIG_BLOB_NAME} + + # Build per-arch OCI image manifest by substituting all four placeholders + sed -e "s/CONFIG_DIGEST/${CONFIG_BLOB_NAME}/g" \ + -e "s/CONFIG_SIZE/${CONFIG_SIZE}/g" \ + -e "s/LAYER_DIGEST/${LAYER_DIGEST}/g" \ + -e "s/LAYER_SIZE/${LAYER_SIZE}/g" \ + artifacts/config/config-multi.json > /tmp/manifest_${ARCH}.json + + MANIFEST_DIGEST=$(sha256sum /tmp/manifest_${ARCH}.json | cut -d ' ' -f 1) + MANIFEST_SIZE=$(stat --printf="%s" /tmp/manifest_${ARCH}.json) + cp /tmp/manifest_${ARCH}.json release-payload-multi/blobs/sha256/${MANIFEST_DIGEST} + + # Accumulate manifest entry for OCI index + ENTRY="{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"digest\":\"sha256:${MANIFEST_DIGEST}\",\"size\":${MANIFEST_SIZE},\"platform\":{\"os\":\"linux\",\"architecture\":\"${ARCH}\"}}" + if [ -z "${MANIFESTS_JSON}" ]; then + MANIFESTS_JSON="${ENTRY}" + else + MANIFESTS_JSON="${MANIFESTS_JSON},${ENTRY}" + fi +done + +# Step 3: Push each per-arch image to the registry with an arch-specific tag, +# then assemble a manifest list using podman manifest. +for ARCH in amd64 arm64 s390x ppc64le; do + CONFIG_BLOB_NAME="${ARCH_CONFIG_BLOB[$ARCH]}" + ARCH_MANIFEST_FILE="/tmp/manifest_${ARCH}.json" + ARCH_MANIFEST_DIGEST=$(sha256sum "${ARCH_MANIFEST_FILE}" | cut -d ' ' -f 1) + ARCH_MANIFEST_SIZE=$(stat --printf="%s" "${ARCH_MANIFEST_FILE}") + + # Build a single-arch OCI layout for this arch + ARCH_DIR="/tmp/oci_${ARCH}" + rm -rf "${ARCH_DIR}" + mkdir -p "${ARCH_DIR}/blobs/sha256" + echo '{"imageLayoutVersion": "1.0.0"}' > "${ARCH_DIR}/oci-layout" + + cp release-payload-multi/blobs/sha256/${LAYER_DIGEST} "${ARCH_DIR}/blobs/sha256/" + cp release-payload-multi/blobs/sha256/${CONFIG_BLOB_NAME} "${ARCH_DIR}/blobs/sha256/" + cp "${ARCH_MANIFEST_FILE}" "${ARCH_DIR}/blobs/sha256/${ARCH_MANIFEST_DIGEST}" + + cat > "${ARCH_DIR}/index.json" </dev/null || true +podman manifest create ${REPO}:${TAG} +for ARCH in amd64 arm64 s390x ppc64le; do + podman manifest add --arch=${ARCH} --os=linux ${REPO}:${TAG} docker://${REPO}:${TAG}-${ARCH} +done + +# Push the manifest list to the registry +podman manifest push --all --remove-signatures ${REPO}:${TAG} docker://${REPO}:${TAG} + +# Save the manifest index for committing (used by generate-release-signature.sh) +skopeo inspect --raw docker://${REPO}:${TAG} > release-payload-multi/index.json + +# Ensure oci-layout marker exists +cat > release-payload-multi/oci-layout <<'EOF' +{"imageLayoutVersion": "1.0.0"} +EOF + +# Clean up staging +rm -rf artifacts/staging/* + +echo "" +echo "Successfully pushed ${REPO}:${TAG}" +echo "" +echo "Next steps:" +echo " make sign-multi (cosign tag-based signature)" +echo " ./generate-release-signature-multi.sh (GPG atomic container signature)" +echo " cp keys/* ../../testdata/keys/" +echo " Commit: release-payload-multi/index.json, ../../testdata/keys/release-pk.asc, ../../testdata/keys/v0.0.1-sha256-*" diff --git a/tests/integration/image-builders/release/generate-release-signature-multi.sh b/tests/integration/image-builders/release/generate-release-signature-multi.sh new file mode 100755 index 000000000..0affedee0 --- /dev/null +++ b/tests/integration/image-builders/release/generate-release-signature-multi.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Generate a throwaway GPG keypair, sign both the single-arch and multi-arch +# test release image digests (atomic container signature JSON), +# and write only the public key plus signatures under keys/. +# No registry or credentials required. +# Requires: gpg, jq. Paths are relative to the integration repo root. +# +# Both images are signed with the same key so that a single release-pk.asc +# validates all signatures. Run this script after building both release images. +# For single-arch only, use generate-release-signature.sh instead. +# +# ./image-builders/release/generate-release-signature-multi.sh + +set -euo pipefail + +if [[ $# -gt 0 ]]; then + echo "Usage: $0" >&2 + echo " (no arguments — signs both releases, writes to keys/)" >&2 + exit 1 +fi + +command -v gpg >/dev/null || { + echo "gpg is required" >&2 + exit 1 +} +command -v jq >/dev/null || { + echo "jq is required" >&2 + exit 1 +} + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KEYS_DIR="${REPO_ROOT}/keys" +GPG_IDENTITY="robot@test.com" + +TMP_GNUPG="$(mktemp -d)" +cleanup() { + rm -rf "${TMP_GNUPG}" +} +trap cleanup EXIT +export GNUPGHOME="${TMP_GNUPG}" +gpg --batch --passphrase '' --pinentry-mode loopback \ + --quick-generate-key "oc-mirror Release Test <${GPG_IDENTITY}>" rsa4096 default 0 + +mkdir -p "${KEYS_DIR}" +PK_OUT="${KEYS_DIR}/release-pk.asc" + +sign_release() { + local index_json="$1" + local image_ref="$2" + local use_file_digest="${3:-false}" + + if [[ ! -f "${index_json}" ]]; then + echo "Skipping ${image_ref}: ${index_json} not found." >&2 + return 0 + fi + + local digest + if [[ "${use_file_digest}" == "true" ]]; then + # Multi-arch: the manifest list digest IS the sha256 of the index.json blob + digest="sha256:$(sha256sum "${index_json}" | cut -d ' ' -f 1)" + else + digest="$(jq -r ' + ([.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64") | .digest][0]) + // .manifests[0].digest + ' "${index_json}")" + if [[ -z "${digest}" || "${digest}" == "null" || "${digest}" != sha256:* ]]; then + echo "Could not read a valid manifest digest from ${index_json}" >&2 + exit 1 + fi + fi + + local digest_hex="${digest#sha256:}" + local tag="${image_ref##*:}" + + local sig_payload + sig_payload="$(jq -nc \ + --arg ref "${image_ref}" \ + --arg dig "${digest}" \ + '{critical:{identity:{"docker-reference":$ref},image:{"docker-manifest-digest":$dig},type:"atomic container signature"},optional:{}}')" + + local sig_file="${KEYS_DIR}/${tag}-sha256-${digest_hex}" + + rm -f "${sig_file}" + printf '%s' "${sig_payload}" | gpg --batch \ + --output "${sig_file}" \ + --local-user "${GPG_IDENTITY}" \ + --digest-algo SHA512 \ + --sign + + if [[ ! -f "${sig_file}" ]]; then + echo "Signing failed — ${sig_file} was not created" >&2 + exit 1 + fi + + echo "Wrote ${sig_file}" +} + +# Single-arch release +sign_release \ + "${REPO_ROOT}/image-builders/release/release-payload/index.json" \ + "quay.io/oc-mirror/release/test-release-index:v0.0.1" + +# Multi-arch release (the manifest list digest is the sha256 of the index.json blob itself) +sign_release \ + "${REPO_ROOT}/image-builders/release/release-payload-multi/index.json" \ + "quay.io/oc-mirror/release/test-release-index-multi:v0.0.1" \ + "true" + +gpg -a --export --output "${PK_OUT}" "${GPG_IDENTITY}" +echo "Wrote ${PK_OUT}" diff --git a/tests/integration/image-builders/release/release-payload-multi/index.json b/tests/integration/image-builders/release/release-payload-multi/index.json new file mode 100644 index 000000000..dd978c586 --- /dev/null +++ b/tests/integration/image-builders/release/release-payload-multi/index.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:20d2c230f7dd11aaf428f4700e1f69e9550924dbdf44541b81dbbffb72823f6c","size":478,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:ba59faf3e9eaee6d1b5b4eebf82eaa59427821e38ba45b4ee8d826fe7e1c34a2","size":478,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:e7bdeb0c63b498ea60330a415dcae2a7da28b0cd7366988b8e363c9cb19d1c46","size":478,"platform":{"architecture":"s390x","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:5f1084981de2ef9abcad4099fac1ea692b1896ae795f1890ff0c2b617ae0c61a","size":478,"platform":{"architecture":"ppc64le","os":"linux"}}]} \ No newline at end of file diff --git a/tests/integration/image-builders/release/release-payload-multi/oci-layout b/tests/integration/image-builders/release/release-payload-multi/oci-layout new file mode 100644 index 000000000..60304f2b2 --- /dev/null +++ b/tests/integration/image-builders/release/release-payload-multi/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion": "1.0.0"} diff --git a/tests/integration/keys/release-pk.asc b/tests/integration/keys/release-pk.asc new file mode 100644 index 000000000..788d0f864 --- /dev/null +++ b/tests/integration/keys/release-pk.asc @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGpaNDoBEADXPpAbD/WzpFGynxxjA+m1YhZdQvIE/wn41rwrc2JjCuKSgDaj +Ffdzfex0IYyO7iXKb/0lBVFxt0fppnbRwLxnLDKi10jB3ofjuZjsEUPi77DgLMo/ +BsVSfAGmIGS9d6je7cmKCNPJyzXHNg1YcrMYvD15wyRj8XKc/YettZzfhUpLVpUP +qW0c88QryMwniehD2qhQoL/y3aUOgoeKAqPC0YwbUsb/cJuWcW2sQCw0lYAhRkZD +xoNV6ZbF/40JRiazt2Ropj7Q7EntEv+I/PZL90fuA9uOu+vcXE7nbd9YPLBV3gC+ +IyHE/fo/iNU0Fu27HCqE1kPA1PdVnDgs1+ungEOib0YCqtGw1Je5zkaqRDtn3JTp +504kAcmYZ8aFOf3JTvpUy+W3q1BLhQyLuSPi3RcSHIeJF0iyh/iQuUKf1wd2q/MP +/25yeFDF/HDu6izTvdXqQNXjNgqlVWCRtCGhL407Lm3DcrgX/O2zd4SqMAqIn0Bv +2pBu/lGz5R16yp7eVRt5p1iNjrVs6COzBWaTACl2j0uSGmvhx0DWN7htnYvFzypP +OzE51si9dx+cblGW1ZbIPPhAZjQthYD7vVqLXQWltTQyRiBrsLfJ9cepxMBmJHQ1 +wk9GVWkRouweXvaXQvsmRErwEPo2I4ldT77Sqy/WXw4D8RkKTWqz3gh/KQARAQAB +tCdvYy1taXJyb3IgUmVsZWFzZSBUZXN0IDxyb2JvdEB0ZXN0LmNvbT6JAlEEEwEK +ADsWIQTHUZvnODbMUfzWroboWcYtHlC9vAUCalo0OgIbAwULCQgHAgIiAgYVCgkI +CwIEFgIDAQIeBwIXgAAKCRDoWcYtHlC9vPWwD/9EL1F+4e9eU6z4AcRO6mRH1Pmk +0eawjKyXMmexdQVF68v46dPqoR8WwqddGGyLsmplM2VtxXheho8R0vRpddb2MM6F +JaVJ8V95wlO99GA/UCv/7x56p8gtjmuufNoEufxUgKgkS/7Q4ef8Z1rCp6wlL+JY +ElkAxMs9QQqBjHzYE0hHyGGEXnd/XpIIsz+urzQA2OdaNZ5vnIjfRP3tPxvCaTNw +Y5k1wMvEx1x295o2ujSLv4Bf0R6WoNtHfJDwm3uqgETqWRIUBvM0SXAJwVLJSz0B +7yoPB7bWXl7mtXctE3XCr3t29VlfybZkHO6+5jumdRQfgGJqbHshzgMZDeWWAUH0 +VuqOun1k7KBrszSariO+8kbgTZVIs7obiWk5hO0vtaH07uyPow/gcJVC7h+6I0ff +rGo/B0nlpKaTBf8ogTmG8NCS6EEG5gLeCVrf8mRRkXdauBFKh7z2yFtU83CSJo05 +XjtIjiMaJfhe4nz2A8rSBeRjqvSwKJDZn4ob7sOwQIDIDDxZKq/bj8flru0EZzkV +9+vfONPiJ5gZuJNt2Ug3yCLi3Zzq4bmDV3cBhPTytS4XCFHgt852LU9rAADp4h9Z +XYxbAI5agrlkz6bAl3SS3NUnFD+YH22s0DecWotwQ1/OJEL9qKshj6B9QWqd8Akq +IbMmz1SqVng3U1Du+w== +=fmlz +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/integration/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 b/tests/integration/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 new file mode 100644 index 000000000..d24dfe653 Binary files /dev/null and b/tests/integration/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 differ diff --git a/tests/integration/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b b/tests/integration/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b new file mode 100644 index 000000000..c169109eb Binary files /dev/null and b/tests/integration/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b differ diff --git a/tests/integration/sparse_manifest_helpers_test.go b/tests/integration/sparse_manifest_helpers_test.go new file mode 100644 index 000000000..f14d77a36 --- /dev/null +++ b/tests/integration/sparse_manifest_helpers_test.go @@ -0,0 +1,95 @@ +package integration_test + +import ( + "context" + "fmt" + "net/http" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/openshift/oc-mirror/tests/integration/pkg/registry" +) + +// expectOnlyPlatformBlobsPresent verifies that only the specified platforms have +// reachable blobs in the registry for the given manifest-list image. +// Other platforms may appear in the index but their blobs must be absent (HTTP 404). +func expectOnlyPlatformBlobsPresent(ctx context.Context, reg *registry.Registry, repo, tag string, expectedPlatforms []string) { + GinkgoHelper() + present, missing := getPlatformBlobPresence(ctx, reg, repo, tag) + for _, exp := range expectedPlatforms { + Expect(present).To(ContainElement(exp), + "expected platform %q to have a blob in the registry", exp) + } + for _, got := range present { + matched := false + for _, exp := range expectedPlatforms { + if got == exp || strings.HasPrefix(got, exp+"/") { + matched = true + break + } + } + Expect(matched).To(BeTrue(), + "unexpected platform %q has a blob in the registry (should be filtered out)", got) + } + _ = missing // platforms in index but no blob — expected for sparse manifests +} + +// expectAllPlatformBlobsPresent verifies that every platform entry in the +// manifest list has a reachable blob (no sparse filtering applied). +func expectAllPlatformBlobsPresent(ctx context.Context, reg *registry.Registry, repo, tag string) { + GinkgoHelper() + _, missing := getPlatformBlobPresence(ctx, reg, repo, tag) + Expect(missing).To(BeEmpty(), + "expected all platform blobs to be present, but these are missing: %v", missing) +} + +// getPlatformBlobPresence returns two lists: platforms with blobs present and +// platforms whose blobs are missing (listed in index but not stored). +func getPlatformBlobPresence(ctx context.Context, reg *registry.Registry, repo, tag string) (present, missing []string) { + GinkgoHelper() + + refStr := fmt.Sprintf("%s/%s:%s", reg.Endpoint(), repo, tag) + ref, err := name.ParseReference(refStr, name.Insecure) + Expect(err).NotTo(HaveOccurred()) + + idx, err := remote.Index(ref, + remote.WithContext(ctx), + remote.WithAuth(authn.Anonymous)) + Expect(err).NotTo(HaveOccurred(), "expected a manifest list at %s", refStr) + + manifest, err := idx.IndexManifest() + Expect(err).NotTo(HaveOccurred()) + Expect(manifest.Manifests).NotTo(BeEmpty(), "manifest list has no entries") + + for _, m := range manifest.Manifests { + if m.Platform == nil { + continue + } + plat := m.Platform.OS + "/" + m.Platform.Architecture + if m.Platform.Variant != "" { + plat += "/" + m.Platform.Variant + } + + // Check if the per-platform manifest blob is reachable. + manifestURL := fmt.Sprintf("http://%s/v2/%s/manifests/%s", reg.Endpoint(), repo, m.Digest.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestURL, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json") + + resp, err := http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + present = append(present, plat) + } else { + missing = append(missing, plat) + } + } + return present, missing +} diff --git a/tests/integration/sparse_manifest_test.go b/tests/integration/sparse_manifest_test.go new file mode 100644 index 000000000..55eb7e0b8 --- /dev/null +++ b/tests/integration/sparse_manifest_test.go @@ -0,0 +1,324 @@ +package integration_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openshift/oc-mirror/tests/integration/pkg/ocmirror" + "github.com/openshift/oc-mirror/tests/integration/pkg/registry" +) + +var _ = Describe("sparse manifest platform filtering", Label("sparse-manifest"), func() { + var workDir string + + BeforeEach(func() { workDir = setupWorkDir() }) + AfterEach(func() { cleanupWorkDir(workDir) }) + + Describe("config validation", func() { + It("should fail when both platform.platforms and platform.architectures are set", func() { + By("running mirrorToMirror with an ISC that uses both platforms and architectures") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-platforms-and-archs.yaml"), + workDir, testRegistry.Endpoint(), + "--dest-tls-verify=false") + By("verifying oc-mirror exits with an error containing the mutual-exclusivity message") + expectOcMirrorExitCode(result, err, 1, + "platform.platforms and platform.architectures cannot be used together") + }) + }) + + Describe("M2M", func() { + Context("with platforms filter (sparse manifests)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should mirror only declared arch blobs for each image", func() { + By("mirroring with platforms: [{linux, amd64}, {linux, arm64}] on multi-platform-container and no filter on empty-image") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-mixed.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying empty-image was mirrored without filtering") + expectRepositoriesExist(*sparseRegistry, []string{"rh_ee_aguidi/empty-image"}) + + By("verifying the release image was mirrored with platform filtering") + expectRepositoriesExist(*sparseRegistry, []string{"openshift/release-images"}) + }) + }) + + Context("with platforms filter — operators, helm, and additionalImages (M2M)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should mirror only declared arch blobs for each content type", func() { + By("mirroring with platforms filter on additionalImages, operators, and helm") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-full.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying operator catalog and helm repositories exist") + expectRepositoriesExist(*sparseRegistry, []string{"oc-mirror/oc-mirror-dev", "podinfo"}) + }) + }) + + Context("without platforms filter (backward compat)", func() { + It("should mirror all blobs without sparse filtering", func() { + By("mirroring without platforms filter") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-no-platforms.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying all platform blobs are present for multi-platform-container") + expectAllPlatformBlobsPresent(ctx, testRegistry, + "rh_ee_aguidi/multi-platform-container", "latest") + + By("verifying the release image was mirrored") + expectRepositoriesExist(*testRegistry, []string{"openshift/release-images"}) + }) + }) + }) + + Describe("M2D + D2M", func() { + Context("with platforms filter (sparse manifests)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should preserve platform filtering through disk archive", func() { + By("mirroring to disk with platforms filter") + result, err := runner.MirrorToDisk(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-mixed.yaml"), + workDir, "--remove-signatures=true") + expectOcMirrorCommandSuccess(result, err) + + By("mirroring from disk to sparse registry") + result, err = runner.DiskToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-mixed.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying the release image was mirrored") + expectRepositoriesExist(*sparseRegistry, []string{"openshift/release-images"}) + }) + }) + + Context("with platforms filter — operators, helm, and additionalImages (M2D+D2M)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should preserve platform filtering through disk archive for all content types", func() { + By("mirroring to disk with platforms filter on operators, helm, and additionalImages") + result, err := runner.MirrorToDisk(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-full.yaml"), + workDir, "--remove-signatures=true") + expectOcMirrorCommandSuccess(result, err) + + By("mirroring from disk to sparse registry") + result, err = runner.DiskToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-full.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying operator catalog and helm repositories exist") + expectRepositoriesExist(*sparseRegistry, []string{"oc-mirror/oc-mirror-dev", "podinfo"}) + }) + }) + + Context("without platforms filter (backward compat)", func() { + It("should preserve all blobs through disk archive", func() { + By("mirroring to disk without platforms filter") + result, err := runner.MirrorToDisk(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-no-platforms.yaml"), + workDir, "--remove-signatures=true") + expectOcMirrorCommandSuccess(result, err) + + By("mirroring from disk to registry") + result, err = runner.DiskToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-no-platforms.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying all platform blobs are present for multi-platform-container") + expectAllPlatformBlobsPresent(ctx, testRegistry, + "rh_ee_aguidi/multi-platform-container", "latest") + + By("verifying the release image was mirrored") + expectRepositoriesExist(*testRegistry, []string{"openshift/release-images"}) + }) + }) + }) +}) + +var _ = Describe("deprecated platform.architectures field", Label("sparse-manifest"), func() { + var workDir string + + BeforeEach(func() { workDir = setupWorkDir() }) + AfterEach(func() { cleanupWorkDir(workDir) }) + + It("should mirror successfully when architectures includes 'multi'", func() { + By("mirroring with deprecated architectures: [multi, amd64]") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-archs-multi.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying multi-platform-container repository exists") + expectRepositoriesExist(*testRegistry, []string{"rh_ee_aguidi/multi-platform-container"}) + }) +}) + +var _ = Describe("negative", Label("sparse-manifest"), func() { + var workDir string + + BeforeEach(func() { workDir = setupWorkDir() }) + AfterEach(func() { cleanupWorkDir(workDir) }) + + Context("strict registry rejects sparse manifest", func() { + It("should mirror with sparse filtering even to a standard registry (no platforms:none required for push)", func() { + By("mirroring with amd64-only filter to a standard registry") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-additional-amd64.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + // The in-process distribution registry v3.1.0 does not enforce manifest index + // blob presence by default, so the push succeeds. Strict validation only applies + // when running a real registry with platforms:none disabled — tested in the manual plan. + expectOcMirrorCommandSuccess(result, err) + }) + }) +}) + +var _ = Describe("deprecated platform.architectures field with release channel", Label("sparse-manifest"), Ordered, func() { + var workDir string + var archRunner *ocmirror.Runner + var cincinnatiServer *httptest.Server + + BeforeAll(func() { + workDir = setupWorkDir() + // Mock Cincinnati server: returns different release payloads depending on ?arch= + cincinnatiServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Accept") != "application/json" { + w.WriteHeader(http.StatusNotAcceptable) + return + } + arch := r.URL.Query().Get("arch") + var payload string + switch arch { + case "multi": + // Use digest reference so GenerateReleaseSignatures can find the GPG signature + payload = "quay.io/oc-mirror/release/test-release-index-multi@sha256:e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83" + default: // amd64 and others — use digest reference for GPG signature lookup + payload = "quay.io/oc-mirror/release/test-release-index@sha256:f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b" + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"nodes":[{"version":"4.18.1","payload":"%s"}],"edges":[]}`, payload) + })) + binaryPath := os.Getenv("OC_MIRROR_BINARY") + archRunner = ocmirror.NewRunner(binaryPath). + WithEnv([]string{fmt.Sprintf("UPDATE_URL_OVERRIDE=%s", cincinnatiServer.URL)}) + }) + + AfterAll(func() { + cincinnatiServer.Close() + cleanupWorkDir(workDir) + }) + + It("should make two Cincinnati calls (multi and amd64) and log deprecation warning", func() { + By("mirroring with deprecated architectures [multi, amd64] and a release channel") + result, err := archRunner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-archs-multi-release.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying the deprecation warning was printed") + Expect(result.Stdout + result.Stderr).To(ContainSubstring( + "platform.architectures is deprecated")) + + By("verifying both release payloads and the additional image are in the registry") + expectRepositoriesExist(*testRegistry, []string{ + "openshift/release-images", + "rh_ee_aguidi/multi-platform-container", + }) + }) +}) diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-amd64.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-amd64.yaml new file mode 100644 index 000000000..5ac909ed9 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-amd64.yaml @@ -0,0 +1,8 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-no-platforms.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-no-platforms.yaml new file mode 100644 index 000000000..ce9843034 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-no-platforms.yaml @@ -0,0 +1,5 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi-release.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi-release.yaml new file mode 100644 index 000000000..242990690 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi-release.yaml @@ -0,0 +1,13 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + architectures: + - multi + - amd64 + channels: + - name: stable-4.18 + minVersion: 4.18.1 + maxVersion: 4.18.1 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi.yaml new file mode 100644 index 000000000..505690e6b --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi.yaml @@ -0,0 +1,9 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + architectures: + - multi + - amd64 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-platforms-and-archs.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-platforms-and-archs.yaml new file mode 100644 index 000000000..1da4e7561 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-platforms-and-archs.yaml @@ -0,0 +1,17 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 + platform: + platforms: + - os: linux + architecture: amd64 + architectures: [arm64] + channels: + - name: stable-4.18 + minVersion: 4.18.1 + maxVersion: 4.18.1 diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-full.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-full.yaml new file mode 100644 index 000000000..6e8097ab0 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-full.yaml @@ -0,0 +1,37 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + graph: false + release: quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + - name: quay.io/rh_ee_aguidi/empty-image:latest + operators: + - catalog: quay.io/oc-mirror/oc-mirror-dev:test-catalog-latest + packages: + - name: foo + - name: bar + platforms: + - os: linux + architecture: amd64 + helm: + repositories: + - name: podinfo + url: https://stefanprodan.github.io/podinfo + charts: + - name: podinfo + version: 5.0.0 + platforms: + - os: linux + architecture: amd64 diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-mixed.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-mixed.yaml new file mode 100644 index 000000000..94a9588ab --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-mixed.yaml @@ -0,0 +1,19 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + graph: false + release: quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + - name: quay.io/rh_ee_aguidi/empty-image:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-no-platforms.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-no-platforms.yaml new file mode 100644 index 000000000..affcb0d29 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-no-platforms.yaml @@ -0,0 +1,9 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + graph: false + release: quay.io/oc-mirror/release/test-release-index:v0.0.1 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + - name: quay.io/rh_ee_aguidi/empty-image:latest diff --git a/tests/integration/testdata/keys/release-pk.asc b/tests/integration/testdata/keys/release-pk.asc index bc453af78..091c1e39a 100644 --- a/tests/integration/testdata/keys/release-pk.asc +++ b/tests/integration/testdata/keys/release-pk.asc @@ -1,29 +1,29 @@ -----BEGIN PGP PUBLIC KEY BLOCK----- -mQINBGnOP5sBEACvf2g92MtrmCMO4rLqegkbOe3wFAD75w2TdcLgdqbrvoVNZPN7 -BJV/AVk02enPSt57EXxoO+p2XFCpmoGQ2Og7qqiJ1Rkoh5fbaKdeVPI3ORXRhMDf -h+aiI1XopdozmQ4b3iGqkrBw3xW6gIUwMc0QSYCf6kyJNLuXhzm5G3xp7PoDiCVv -KQH0/wcY4f+G+bAiq9P8pmLN2i4vNMKo58DDClaLU0WY+epsrh+Dbv7pz0ER/yrp -tZS7hVM16w9Q5Mc1LFLO4tTqWpnNFvjs/C1ITnz/m2cHArG/VySztg0JbJi/Bxw4 -ULz39F0hNNSoGjUWtEG21SvnCnpZZdeAJMPpb6ERZGfNmVQBEmzP+KBofny66DEI -D/tBg6LeEwSrHgsTmawE6SO/w4eSeoMXfIe119bwzj92AuHyFKtBo9LggQHVRc+j -EJX8o0C9h1+ioohkDUZUugKeF3MSEKdW7aF5xveFWi5GroPC4yb0SWu04xTzNWNi -ooAVouI4mv0QgsDi9Hi8lK5KHONrCLTurfwsZkAp7jLWruApwPc5YFMbSTafF3X/ -FCXTaZlMhztxs5MHpLfCDU4o84ye025cVeZZGJc73vVLvF8VJh+PPDbx/pfh8+sZ -Yzz4GZFP4pYywqizOe9PgE//zvKfrWU5nI1aHzddfPFpCrFS/TxirSd3KwARAQAB +mQINBGpaEBIBEADCYhe4nf/NnfNwMkGfwQPrOXqTdN4zt8bCzADIWXZSuyarBEdQ +K5CeYxQmWVtPjCk6jQU4p4OopTS+UmXUXU1pm01ken6jiXmbjLVu2cJItvHxa6B6 +Kt2jbmxzfL/o4FeMgtFGYbsicYSLPy6Np6VAkZzNvxHedgjDLy8+Nag3eelD2JWl +qN8sXL2xzWJu6Dx1EUQyv5eG601nIRmpw/3IZYWg2Urw98YMG00IA+EeQGhzXZQ4 +hskYFF+Q7bRdGL07SbMv6RN/m5p9eQnjGPLz73iAZfd5N3v6T/f2mdLGW2pVyjsF +b7Dkdn/q1+pcreKMJctryYXE1rmHJvP4eH/w2eWh0vl/cKeH87Xq7FjNfpEXXefq +6tAW8XtzCEY6fL8Wgk0DgFbaYWdIqngX+N/XSjWHwgSD3VTNSC+Gb2DIbxYNi4vt +LrSrKifjN2NyokHqfvzoHZBl8+KlAB7OtcipVqUqaLZSeJm2L6/goNCQG5VvqRT6 +ConctExIAc49vYJVeqeeHYIUDqz6AlGV2MnVDFGTwn2CSY/VlWdSInyhqVRnfQ8L +laqgV38p3dj0apWqY3qe1VvpHOCZExYjnfSr3RUO9h2je9F6Gi72o4ZOmNLZCCeR +rborQUXlEwrnHat+ZNJqpiT33nVCnfbLK+kihlG9sKggIg6vHxu0AZb3uwARAQAB tCdvYy1taXJyb3IgUmVsZWFzZSBUZXN0IDxyb2JvdEB0ZXN0LmNvbT6JAlEEEwEK -ADsWIQTJNjGZurFS25ebUF+ldt2a9Y2GMQUCac4/mwIbAwULCQgHAgIiAgYVCgkI -CwIEFgIDAQIeBwIXgAAKCRCldt2a9Y2GMYC2D/9rP7Dv3JkflnTp7K+LJMeohLHA -aR3S1MiAiz8fDGWN4cz/OVmUhMv5q7q3zwP5Ivux18QzH7CPKPBTTxHOUmUBICLm -lkObLW8Lrk3MqoqbeRpM3kUMZLZLPGqNPPRJlr4SK5ak92XZJQZ0xLBblmmPiqNE -6zIIEkdWHc2hAHiTDTctUBO7NLMUXJRxhp+xDW5Tuam/PKjYJ4UIT/2EeoWb+gUx -k6oRDMDUnaXWr92VyhJcEEzEHI7FoJaoXzxH8yvK4RFclhc/1WloriN8iU0IK8QD -zOhmWeCDM5Djxz0HAxEu+DuQ4sQ6qRonzvjWHmKfBwIEvkR7LTqR0qH2DGECYBhZ -G+Yub10CfFLNz+aKw8A6te/ms/oMCbb4/s5uRIG7XyCUKx3Tri59inBt/sALDdIh -uI/TJqQZ35wS7jUMsL87Ee/BZiRZW6cMVC9QHO4Gu6U7VIyHUGbnZnV6wHaI2D8l -D7lQwNLXZL876S4i9KI2IheEfwWBNU9mFp3Giyh1lFdSaoiNVJFVwETF0bWxXtCR -+vFfPbsuQrCz65AxHl+tgA3RnCO4EItJ4eoGnm/uo6tRa8iANKx0nFcISi2Piipz -5ledtBXYN/Be5Ei9ASNCxvKki8o6X+WGDhB2qI0dmUSSAz485Ykz7++hrqfI4+X6 -d4lZiUlY/UKrPHlMsw== -=ZVO9 +ADsWIQSGmkqy5QIlOlUVNKf5ltNto5RokgUCaloQEgIbAwULCQgHAgIiAgYVCgkI +CwIEFgIDAQIeBwIXgAAKCRD5ltNto5RokmKGEACoIT4+zDU2qZPsT+Mh0XUiwpM9 +RkRS2AM4x9Fp5DcN6UsHP51IaATlM1K68TliBKI6RvMtYGhizQ06ZH8wVWtSFUsC +lI/9ujZ9cUpwUKMQAK4zSlIJ1LnqA7drpzlSy9PjK2cckiwgF3HzbKqkmTMYWvON +XlxY+8noDH5eOG4ALgVR4lEsYZFayFEpAmQckVV9eMfMrU1b00x7gd7PS/a6yFlk +JlwPbhvhb5jucZUuH4jV/hVTaUJ1zweLEzRkH66q1d5p+3gIBvMWWMe4X9ZGZnGb +tNiQf2fcwNi9Z8mHqZHOOfCxGRASdAvBuXKqIjWOnkp5h66bEYssuGK88Lywq2Do +OhTVSLJofc1jrXgPm1J0qcMghCpMg3/8OTDUxaKD+oYyggpa/O2E3dRRBsly4s6D +Oo1Mf+stGEP/Od+P3J00puPWfeK78zrKKxDF5JX8du0lgP4wusj/PG6mtDdO560g +7oq2ozay02/ajwdJpVknlnOR9eeZa4piU+hP4oUFQVeVdQ1nG7X9FspPu50Pg8hS +be2TgsG4SxMOfkzT46v1bMZAW64ScaipA8xegGKyUA24VanimgFqYdYuiTiOyoKR +GjWyA5nMEvuZgis+I0RQO4Rq0m7CL/7JIdZsRODk12FVuyce8bCLu+Lreg6s6z+z +7gzfP9SLEHQQ4dlZrg== +=h3in -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/integration/testdata/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 b/tests/integration/testdata/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 new file mode 100644 index 000000000..9c4889ae0 Binary files /dev/null and b/tests/integration/testdata/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 differ diff --git a/tests/integration/testdata/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b b/tests/integration/testdata/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b index e70d72a6d..0ee72c7c7 100644 Binary files a/tests/integration/testdata/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b and b/tests/integration/testdata/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b differ diff --git a/tests/integration/testdata/registry-config-sparse.yaml b/tests/integration/testdata/registry-config-sparse.yaml new file mode 100644 index 000000000..bb9863012 --- /dev/null +++ b/tests/integration/testdata/registry-config-sparse.yaml @@ -0,0 +1,27 @@ +version: 0.1 +log: + accesslog: + disabled: true + level: error + fields: + service: registry +storage: + delete: + enabled: true + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /tmp/ +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +validation: + manifests: + indexes: + platforms: none