diff --git a/docs/features/custom-release-repo-paths.md b/docs/features/custom-release-repo-paths.md new file mode 100644 index 000000000..053c73d36 --- /dev/null +++ b/docs/features/custom-release-repo-paths.md @@ -0,0 +1,89 @@ +# Custom Release Repository Paths + +## Overview + +By default, oc-mirror copies release images to `openshift/release-images` and release component images to `openshift/release` on the destination registry. The `releaseImageRepo` and `releaseComponentRepo` fields in the ImageSetConfiguration allow you to override these destination paths, for example to preserve the original source repository paths. + +## Configuration + +Add the fields to the `platform` section of your ImageSetConfiguration: + +```yaml +apiVersion: mirror.openshift.io/v2alpha1 +kind: ImageSetConfiguration +mirror: + platform: + releaseImageRepo: "openshift-release-dev/ocp-release" + releaseComponentRepo: "openshift-release-dev/ocp-v4.0-art-dev" + channels: + - name: stable-4.22 + minVersion: 4.22.0 + maxVersion: 4.22.0 +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `releaseImageRepo` | `openshift/release-images` | Destination repository path for release images | +| `releaseComponentRepo` | `openshift/release` | Destination repository path for release component images | + +Both fields are optional. When omitted, the default paths are used. + +## Example + +Given the following `imageset-config.yaml`: + +```yaml +apiVersion: mirror.openshift.io/v2alpha1 +kind: ImageSetConfiguration +mirror: + platform: + releaseImageRepo: "openshift-release-dev/ocp-release" + releaseComponentRepo: "openshift-release-dev/ocp-v4.0-art-dev" + channels: + - name: stable-4.22 + minVersion: 4.22.0 + maxVersion: 4.22.0 +``` + +Run oc-mirror: + +```bash +oc-mirror --v2 \ + --authfile ./pull-secret.json \ + --config ./isc.yaml \ + --workspace file:///home/user/oc-mirror-workspace \ + docker://registry.example.com/mirrors/quay.io +``` + +With the default configuration, release images would be copied to: + +```text +registry.example.com/mirrors/quay.io/openshift/release-images:4.22.0-x86_64 +registry.example.com/mirrors/quay.io/openshift/release:4.22.0-x86_64- +``` + +With the overrides above, they are instead copied to: + +```text +registry.example.com/mirrors/quay.io/openshift-release-dev/ocp-release:4.22.0-x86_64 +registry.example.com/mirrors/quay.io/openshift-release-dev/ocp-v4.0-art-dev:4.22.0-x86_64- +``` + +## IDMS Configuration + +With the original source paths preserved on the mirror, a single `ImageDigestMirrorSet` entry can cover all of `quay.io`: + +```yaml +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: quay-io-mirror +spec: + imageDigestMirrors: + - source: quay.io + mirrors: + - registry.example.com/mirrors/quay.io + mirrorSourcePolicy: NeverContactSource # optional, for fully disconnected environments +``` + +Because the mirrored paths match the source paths, every image under `quay.io/` is automatically resolved to `registry.example.com/mirrors/quay.io/` with no per-repository mappings needed. diff --git a/docs/okd-mirror.md b/docs/okd-mirror.md index 8925a1c93..5d1ff899b 100644 --- a/docs/okd-mirror.md +++ b/docs/okd-mirror.md @@ -1,18 +1,69 @@ # Mirror OKD -To pass signatures verification when trying to mirror OKD images, you need to set the following env variables: -- OCP_SIGNATURE_URL="https://storage.googleapis.com/openshift-ci-release/releases/signatures/openshift/release/" -- OCP_SIGNATURE_VERIFICATION_PK="/path/to/PK" (recovered from "https://raw.githubusercontent.com/openshift/cluster-update-keys/master/keys/verifier-public-key-openshift-ci-4") +## Signature Verification +To pass signature verification when mirroring OKD images, set the following environment variables: + +```bash +export OCP_SIGNATURE_URL="https://storage.googleapis.com/openshift-ci-release/releases/signatures/openshift/release/" +export OCP_SIGNATURE_VERIFICATION_PK="/path/to/verifier-public-key-openshift-ci-4" ``` + +The public key can be retrieved from https://raw.githubusercontent.com/openshift/cluster-update-keys/master/keys/verifier-public-key-openshift-ci-4 + +Alternatively, use `--ignore-release-signature` to skip release signature verification. + +## Basic Configuration + +OKD channels follow a different naming convention than OCP. Releases are listed at https://origin-release.ci.openshift.org, but note that the channel names differ from the website headings: + +| Website heading | Channel name | +|-----------------|--------------| +| 4-scos-stable | `stable-4-scos` | +| 4-scos-next | `next-4-scos` | +| 5-scos-next | `next-5-scos` | + +The `type: okd` field must be set. + +```yaml +--- apiVersion: mirror.openshift.io/v2alpha1 kind: ImageSetConfiguration mirror: platform: - graph: false channels: - - name: 4-stable - minVersion: 4.18.0-okd-scos.8 - maxVersion: 4.18.0-okd-scos.8 - type: okd -``` \ No newline at end of file + - name: stable-4-scos + minVersion: 4.18.0-okd-scos.8 + maxVersion: 4.18.0-okd-scos.8 + type: okd +``` + +## Custom Repository Paths + +By default, OKD release images are copied to `openshift/release-images` and component images to `openshift/release` on the destination registry. Use `releaseImageRepo` and `releaseComponentRepo` to override these paths, for example to match the source repository layout: + +```yaml +mirror: + platform: + releaseImageRepo: "okd/scos-release" + releaseComponentRepo: "okd/scos-content" + channels: + ... +``` + +Run oc-mirror: + +```bash +oc-mirror --v2 \ + --authfile ./pull-secret.json \ + --config ./imageset-config.yaml \ + --workspace file:///home/user/oc-mirror-workspace \ + docker://registry.example.com/mirrors/quay.io +``` + +With the overrides above, images are copied to: + +``` +registry.example.com/mirrors/quay.io/okd/scos-release:4.18.0-okd-scos.8 +registry.example.com/mirrors/quay.io/okd/scos-content:4.18.0-okd-scos.8- +``` diff --git a/internal/pkg/api/v2alpha1/type_config.go b/internal/pkg/api/v2alpha1/type_config.go index 12b24c3f3..33d50cf42 100644 --- a/internal/pkg/api/v2alpha1/type_config.go +++ b/internal/pkg/api/v2alpha1/type_config.go @@ -102,11 +102,21 @@ type Platform struct { // will be used to extract the kubeVirtContainer image // from the release payload file 0000_50_installer_coreos-bootimages KubeVirtContainer bool `json:"kubeVirtContainer,omitempty"` + // ReleaseImageRepo overrides the destination repository path for + // release images (default: openshift/release-images) + ReleaseImageRepo string `json:"releaseImageRepo,omitempty"` + // ReleaseComponentRepo overrides the destination repository path for + // release component images (default: openshift/release) + ReleaseComponentRepo string `json:"releaseComponentRepo,omitempty"` } func (p Platform) DeepCopy() Platform { platformCopy := Platform{ - Graph: p.Graph, + Graph: p.Graph, + Release: p.Release, + KubeVirtContainer: p.KubeVirtContainer, + ReleaseImageRepo: p.ReleaseImageRepo, + ReleaseComponentRepo: p.ReleaseComponentRepo, } platformCopy.Channels = make([]ReleaseChannel, len(p.Channels)) diff --git a/internal/pkg/cli/executor.go b/internal/pkg/cli/executor.go index e1b88f273..755c385cd 100644 --- a/internal/pkg/cli/executor.go +++ b/internal/pkg/cli/executor.go @@ -708,6 +708,10 @@ http: // setupLocalStorage - private function that sets up // a local (distribution) registry func (o *ExecutorSchema) setupLocalStorage(ctx context.Context) error { + // distribution/distribution parses all REGISTRY_* env vars as config overrides, + // colliding with the Podman/containers REGISTRY_AUTH_FILE convention. + // The value has already been captured as the --authfile default (options.go). + os.Unsetenv("REGISTRY_AUTH_FILE") config, err := o.setupLocalRegistryConfig() if err != nil { o.Log.Error("parsing config %v", err) diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 4cd24261c..51ee811e4 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -337,6 +337,15 @@ func (o *LocalStorageCollector) prepareGraphImage(ctx context.Context) (v2alpha1 } func (o LocalStorageCollector) prepareM2DCopyBatch(images []v2alpha1.RelatedImage, releaseTag string) ([]v2alpha1.CopyImageSchema, error) { + releaseImagePath := releaseImagePathComponents + if o.Config.Mirror.Platform.ReleaseImageRepo != "" { + releaseImagePath = o.Config.Mirror.Platform.ReleaseImageRepo + } + releaseComponentPath := releaseComponentPathComponents + if o.Config.Mirror.Platform.ReleaseComponentRepo != "" { + releaseComponentPath = o.Config.Mirror.Platform.ReleaseComponentRepo + } + result := make([]v2alpha1.CopyImageSchema, 0, len(images)) for _, img := range images { var src string @@ -348,7 +357,7 @@ func (o LocalStorageCollector) prepareM2DCopyBatch(images []v2alpha1.RelatedImag } src = imgSpec.ReferenceWithTransport - pathComponents := preparePathComponents(imgSpec, img.Type, img.Name) + pathComponents := preparePathComponents(imgSpec, img.Type, img.Name, releaseImagePath, releaseComponentPath) tag := prepareTag(imgSpec, img.Type, releaseTag, img.Name) dest = consts.DockerProtocol + strings.Join([]string{o.destinationRegistry(), pathComponents + ":" + tag}, "/") @@ -361,6 +370,15 @@ func (o LocalStorageCollector) prepareM2DCopyBatch(images []v2alpha1.RelatedImag } func (o LocalStorageCollector) prepareD2MCopyBatch(images []v2alpha1.RelatedImage, releaseTag string) ([]v2alpha1.CopyImageSchema, error) { + releaseImagePath := releaseImagePathComponents + if o.Config.Mirror.Platform.ReleaseImageRepo != "" { + releaseImagePath = o.Config.Mirror.Platform.ReleaseImageRepo + } + releaseComponentPath := releaseComponentPathComponents + if o.Config.Mirror.Platform.ReleaseComponentRepo != "" { + releaseComponentPath = o.Config.Mirror.Platform.ReleaseComponentRepo + } + result := make([]v2alpha1.CopyImageSchema, 0, len(images)) for _, img := range images { var src string @@ -371,7 +389,7 @@ func (o LocalStorageCollector) prepareD2MCopyBatch(images []v2alpha1.RelatedImag return nil, err } - pathComponents := preparePathComponents(imgSpec, img.Type, img.Name) + pathComponents := preparePathComponents(imgSpec, img.Type, img.Name, releaseImagePath, releaseComponentPath) tag := prepareTag(imgSpec, img.Type, releaseTag, img.Name) src = consts.DockerProtocol + strings.Join([]string{o.LocalStorageFQDN, pathComponents + ":" + tag}, "/") @@ -581,15 +599,15 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C } } -func preparePathComponents(imgSpec image.ImageSpec, imgType v2alpha1.ImageType, imgName string) string { +func preparePathComponents(imgSpec image.ImageSpec, imgType v2alpha1.ImageType, imgName string, releaseImagePath, releaseComponentPath string) string { pathComponents := "" switch { case imgType == v2alpha1.TypeOCPRelease: - pathComponents = releaseImagePathComponents + pathComponents = releaseImagePath case imgType == v2alpha1.TypeCincinnatiGraph: pathComponents = imgSpec.PathComponent case imgType == v2alpha1.TypeOCPReleaseContent && imgName != "": - pathComponents = releaseComponentPathComponents + pathComponents = releaseComponentPath case imgSpec.IsImageByDigestOnly(): pathComponents = imgSpec.PathComponent } diff --git a/internal/pkg/release/local_stored_collector_test.go b/internal/pkg/release/local_stored_collector_test.go index f29c2ed6d..c797c36cb 100644 --- a/internal/pkg/release/local_stored_collector_test.go +++ b/internal/pkg/release/local_stored_collector_test.go @@ -21,6 +21,7 @@ import ( "github.com/openshift/oc-mirror/v2/internal/pkg/api/v2alpha1" "github.com/openshift/oc-mirror/v2/internal/pkg/consts" "github.com/openshift/oc-mirror/v2/internal/pkg/folder" + "github.com/openshift/oc-mirror/v2/internal/pkg/image" clog "github.com/openshift/oc-mirror/v2/internal/pkg/log" "github.com/openshift/oc-mirror/v2/internal/pkg/mirror" ) @@ -841,3 +842,29 @@ func (o *ManifestMock) ImageDigest(ctx context.Context, sourceCtx *types.SystemC func (o *ManifestMock) ImageManifest(ctx context.Context, sourceCtx *types.SystemContext, imgRef string, instanceDigest *digest.Digest) ([]byte, string, error) { return nil, "", nil } + +func TestPreparePathComponentsWithOverrides(t *testing.T) { + releaseImgSpec, err := image.ParseRef("docker://quay.io/openshift-release-dev/ocp-release:4.13.10-x86_64") + assert.NoError(t, err) + componentImgSpec, err := image.ParseRef("docker://quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e4182331e") + assert.NoError(t, err) + + t.Run("defaults when no overrides", func(t *testing.T) { + assert.Equal(t, releaseImagePathComponents, preparePathComponents(releaseImgSpec, v2alpha1.TypeOCPRelease, "", releaseImagePathComponents, releaseComponentPathComponents)) + assert.Equal(t, releaseComponentPathComponents, preparePathComponents(componentImgSpec, v2alpha1.TypeOCPReleaseContent, "agent-installer-api-server", releaseImagePathComponents, releaseComponentPathComponents)) + }) + + t.Run("release image path override", func(t *testing.T) { + assert.Equal(t, "openshift-release-dev/ocp-release", preparePathComponents(releaseImgSpec, v2alpha1.TypeOCPRelease, "", "openshift-release-dev/ocp-release", releaseComponentPathComponents)) + }) + + t.Run("release component path override", func(t *testing.T) { + assert.Equal(t, "openshift-release-dev/ocp-v4.0-art-dev", preparePathComponents(componentImgSpec, v2alpha1.TypeOCPReleaseContent, "agent-installer-api-server", releaseImagePathComponents, "openshift-release-dev/ocp-v4.0-art-dev")) + }) + + t.Run("graph image unaffected by overrides", func(t *testing.T) { + graphSpec, err := image.ParseRef("docker://localhost:9999/openshift/graph-image:latest") + assert.NoError(t, err) + assert.Equal(t, graphSpec.PathComponent, preparePathComponents(graphSpec, v2alpha1.TypeCincinnatiGraph, "", "override-release", "override-component")) + }) +} diff --git a/internal/pkg/release/signature.go b/internal/pkg/release/signature.go index f0abf96bb..31f93d5d2 100644 --- a/internal/pkg/release/signature.go +++ b/internal/pkg/release/signature.go @@ -80,7 +80,7 @@ func (o SignatureSchema) GenerateReleaseSignatures(ctx context.Context, images [ // OCPBUGS-56009 // not worrying about code complexity as this module will eventually be deprecated // in favor of the cosign signature work - if o.Opts.Global.IgnoreReleaseSignature && len(o.Config.Mirror.Platform.Release) > 0 { + if o.Opts.Global.IgnoreReleaseSignature { imgs = append(imgs, img) continue }