exporter/containerimage: support unpack for rewrite-timestamp mode

client/client_export_image_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/moby/buildkit/util/testutil/integration"
 	"github.com/moby/buildkit/util/testutil/workers"
 	digest "github.com/opencontainers/go-digest"
+	imagespecidentity "github.com/opencontainers/image-spec/identity"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
@@ -175,6 +176,98 @@ func testBuildExportScratch(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, "i am platform linux/arm64", img.Img.Config.Labels["foo"])
 }
 
+// testBuildExportUnpackWithRewriteTimestamp verifies that the "unpack" exporter option
+// can be used together with "rewrite-timestamp", and that the image is unpacked to the
+// snapshot that matches the rewritten layers.
+// https://github.com/moby/buildkit/issues/4230
+func testBuildExportUnpackWithRewriteTimestamp(t *testing.T, sb integration.Sandbox) {
+	workers.CheckFeatureCompat(t, sb, workers.FeatureImageExporter, workers.FeatureSourceDateEpoch)
+	requiresLinux(t)
+
+	cdAddress := sb.ContainerdAddress()
+	if cdAddress == "" {
+		t.Skip("test requires containerd worker")
+	}
+
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	busybox := llb.Image("busybox:latest")
+	cmd := `sh -e -c "echo -n foo > data"`
+
+	st := llb.Scratch()
+	st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	tm := time.Unix(1700000000, 0).UTC()
+	target := "docker.io/buildkit/build/exporter:unpack-rewrite-timestamp"
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type: ExporterImage,
+				Attrs: map[string]string{
+					"name":              target,
+					"unpack":            "true",
+					"rewrite-timestamp": "true",
+					"source-date-epoch": fmt.Sprintf("%d", tm.Unix()),
+				},
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	client, err := newContainerd(cdAddress)
+	require.NoError(t, err)
+	defer client.Close()
+
+	ctx := namespaces.WithNamespace(sb.Context(), "buildkit")
+
+	img, err := client.ImageService().Get(ctx, target)
+	require.NoError(t, err)
+
+	contentStore := client.ContentStore()
+
+	manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default())
+	require.NoError(t, err)
+
+	// all the layers must be rewritten to match the epoch
+	require.NotEmpty(t, manifest.Layers)
+	for _, l := range manifest.Layers {
+		require.Equal(t, fmt.Sprintf("%d", tm.Unix()), l.Annotations["buildkit/rewritten-timestamp"])
+	}
+
+	// the timestamps in the layer blobs must be clamped to the epoch
+	dt, err := content.ReadBlob(ctx, contentStore, manifest.Layers[0])
+	require.NoError(t, err)
+	m, err := testutil.ReadTarToMap(dt, true)
+	require.NoError(t, err)
+	require.Contains(t, m, "data")
+	for name, item := range m {
+		require.False(t, item.Header.ModTime.After(tm), "file %q has timestamp %v, expected not after %v", name, item.Header.ModTime, tm)
+	}
+
+	// the image must be unpacked to the snapshot that matches the rewritten diffIDs
+	dt, err = content.ReadBlob(ctx, contentStore, manifest.Config)
+	require.NoError(t, err)
+	var ociimg ocispecs.Image
+	require.NoError(t, json.Unmarshal(dt, &ociimg))
+	require.NotEmpty(t, ociimg.RootFS.DiffIDs)
+
+	chainID := imagespecidentity.ChainID(ociimg.RootFS.DiffIDs)
+	snapshotService := client.SnapshotService(sb.Snapshotter())
+	_, err = snapshotService.Stat(ctx, chainID.String())
+	require.NoError(t, err)
+
+	err = client.ImageService().Delete(ctx, target, images.SynchronousDelete())
+	require.NoError(t, err)
+
+	checkAllReleasable(t, c, sb, true)
+}
+
 // testBuildExportWithForeignLayer verifies foreign (non-distributable) layer handling during
 // image export. propagate=1 preserves foreign media type and URLs; propagate=0 converts to
 // regular layers. Skipped on Windows: the test image is Linux-only and BuildKit's Windows

client/client_test.go

@@ -63,6 +63,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 
 	// client_export_image_test.go
 	testBuildExportScratch,
+	testBuildExportUnpackWithRewriteTimestamp,
 	testBuildExportWithForeignLayer,
 	testBuildExportWithUncompressed,
 	testBuildExportZstd,

exporter/containerimage/export.go

@@ -27,7 +27,9 @@ import (
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
+	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/llbsolver/compat"
+	"github.com/moby/buildkit/solver/result"
 	"github.com/moby/buildkit/util/compression"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/errutil"
@@ -252,7 +254,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
 		}
 	}()
 
-	desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts, buildInfo.CompatibilityVersion, e.Type())
+	desc, committedRemotes, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts, buildInfo.CompatibilityVersion, e.Type())
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -323,14 +325,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
 				tagDone(nil)
 
 				if e.unpack {
-					if opts.RewriteTimestamp {
-						// e.unpackImage cannot be used because src ref does not point to the rewritten image
-						// /
-						// TODO: change e.unpackImage so that it takes Result[Remote] as parameter.
-						// https://github.com/moby/buildkit/pull/4057#discussion_r1324106088
-						return nil, nil, nil, errors.New("exporter option \"rewrite-timestamp\" conflicts with \"unpack\"")
-					}
-					if err := e.unpackImage(ctx, img, src, session.NewGroup(buildInfo.SessionID)); err != nil {
+					if err := e.unpackImage(ctx, img, src, committedRemotes); err != nil {
 						return nil, nil, nil, err
 					}
 				}
@@ -441,7 +436,7 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou
 	return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, annotations)
 }
 
-func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src *exporter.Source, s session.Group) (err0 error) {
+func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src *exporter.Source, remotes *result.Result[*solver.Remote]) (err0 error) {
 	matcher := platforms.Only(platforms.Normalize(platforms.DefaultSpec()))
 
 	ps, err := exptypes.ParsePlatforms(src.Metadata)
@@ -462,9 +457,9 @@ func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Imag
 		return matcher.Less(matching[i].Platform, matching[j].Platform)
 	})
 
-	ref, _ := src.FindRef(matching[0].ID)
-	if ref == nil {
-		// ref has no layers, so nothing to unpack
+	remote, _ := remotes.FindRef(matching[0].ID)
+	if remote == nil || len(remote.Descriptors) == 0 {
+		// image has no layers, so nothing to unpack
 		return nil
 	}
 
@@ -485,12 +480,6 @@ func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Imag
 		return err
 	}
 
-	remotes, err := ref.GetRemotes(ctx, true, e.opts.RefCfg, false, s)
-	if err != nil {
-		return err
-	}
-	remote := remotes[0]
-
 	// ensure the content for each layer exists locally in case any are lazy
 	if unlazier, ok := remote.Provider.(cache.Unlazier); ok {
 		if err := unlazier.Unlazy(ctx); err != nil {