exporter/containerimage: new option: rewrite-timestamp

rewrite-timestamp rewrites timestamps in layers for reproducible
builds.

When a layer is rewritten, an annotation "buildkit/rewritten-timestamp=<INT64>"
is set to the layer.

Example:
```
buildctl build
  --frontend dockerfile.v0 \
  --opt build-arg:SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} \
  --output type=oci,dest=dest.tar,rewrite-timestamp=true \
  ...
```

Alternative to PR 3560

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

Author: AkihiroSuda

README.md

@@ -261,6 +261,8 @@ Keys supported by image output:
 * `name-canonical=true`: add additional canonical name `name@<digest>`
 * `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz should be used with `oci-mediatypes=true`.
 * `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)
+* `rewrite-timestamp=true` (Present in the `master` branch <!-- TODO: v0.13-->): rewrite the file timestamps to the `SOURCE_DATE_EPOCH` value.
+   See [`docs/build-repro.md`](docs/build-repro.md) for how to specify the `SOURCE_DATE_EPOCH` value.
 * `force-compression=true`: forcefully apply `compression` option to all layers (including already existing layers)
 * `store=true`: store the result images to the worker's (e.g. containerd) image store as well as ensures that the image has all blobs in the content store (default `true`). Ignored if the worker doesn't have image store (e.g. OCI worker).
 * `annotation.<key>=<value>`: attach an annotation with the respective `key` and `value` to the built image

docs/build-repro.md

@@ -57,26 +57,14 @@ The build arg value is used for:
 - the timestamp of the files exported with the `local` exporter
 - the timestamp of the files exported with the `tar` exporter
 
-The build arg value is not used for the timestamps of the files inside the image currently ([Caveats](#caveats)).
-
-See also the [documentation](/frontend/dockerfile/docs/reference.md#buildkit-built-in-build-args) of the Dockerfile frontend.
-
-## Caveats
-### Timestamps of the files inside the image
-Currently, the `SOURCE_DATE_EPOCH` value is not used for the timestamps of the files inside the image.
-
-Workaround:
-```dockerfile
-# Limit the timestamp upper bound to SOURCE_DATE_EPOCH.
-# Workaround for https://github.com/moby/buildkit/issues/3180
-ARG SOURCE_DATE_EPOCH
-RUN find $( ls / | grep -E -v "^(dev|mnt|proc|sys)$" ) -newermt "@${SOURCE_DATE_EPOCH}" -writable -xdev | xargs touch --date="@${SOURCE_DATE_EPOCH}" --no-dereference
-
-# Squashing is needed so that only files with the defined timestamp from the last layer are added to the image.
-# This squashing also addresses non-reproducibility of whiteout timestamps (https://github.com/moby/buildkit/issues/3168) on BuildKit prior to v0.12.
-FROM scratch
-COPY --from=0 / /
+To apply the build arg value to the timestamps of the files inside the image, specify `rewrite-timestamp=true` as an image exporter option:
+```
+--output type=image,name=docker.io/username/image,push=true,rewrite-timestamp=true
 ```
 
-The `touch` command above is [not effective](https://github.com/moby/buildkit/issues/3309) for mount point directories.
-A workaround is to create mount point directories below `/dev` (tmpfs) so that the mount points will not be included in the image layer.
+<!-- TODO: s/master/v0.13/ -->
+The `rewrite-timestamp` option is only available in the `master` branch of BuildKit.
+See [v0.12 documentation](https://github.com/moby/buildkit/blob/v0.12/docs/build-repro.md#caveats) for dealing with timestamps
+in BuildKit v0.12 and v0.11.
+
+See also the [documentation](/frontend/dockerfile/docs/reference.md#buildkit-built-in-build-args) of the Dockerfile frontend.

exporter/containerimage/export.go

@@ -274,6 +274,13 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
 				tagDone(nil)
 
 				if e.unpack {
+					if opts.RewriteTimestamp {
+						// e.unpackImage cannot be used because src ref does not point to the rewritten image
+						///
+						// TODO: change e.unpackImage so that it takes Result[Remote] as parameter.
+						// https://github.com/moby/buildkit/pull/4057#discussion_r1324106088
+						return nil, nil, errors.New("exporter option \"rewrite-timestamp\" conflicts with \"unpack\"")
+					}
 					if err := e.unpackImage(ctx, img, src, session.NewGroup(sessionID)); err != nil {
 						return nil, nil, err
 					}
@@ -310,7 +317,18 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
 				}
 			}
 			if e.push {
-				err := e.pushImage(ctx, src, sessionID, targetName, desc.Digest)
+				if opts.RewriteTimestamp {
+					annotations := map[digest.Digest]map[string]string{}
+					addAnnotations(annotations, *desc)
+					// e.pushImage cannot be used because src ref does not point to the rewritten image
+					//
+					// TODO: change e.pushImage so that it takes Result[Remote] as parameter.
+					// https://github.com/moby/buildkit/pull/4057#discussion_r1324106088
+					err = push.Push(ctx, e.opt.SessionManager, sessionID, e.opt.ImageWriter.opt.ContentStore, e.opt.ImageWriter.ContentStore(),
+						desc.Digest, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, annotations)
+				} else {
+					err = e.pushImage(ctx, src, sessionID, targetName, desc.Digest)
+				}
 				if err != nil {
 					return nil, nil, errors.Wrapf(err, "failed to push %v", targetName)
 				}

exporter/containerimage/exptypes/keys.go

@@ -72,4 +72,8 @@ var (
 	// Value: int (0-9) for gzip and estargz
 	// Value: int (0-22) for zstd
 	OptKeyCompressionLevel ImageExporterOptKey = "compression-level"
+
+	// Rewrite timestamps in layers to match SOURCE_DATE_EPOCH
+	// Value: bool <true|false>
+	OptKeyRewriteTimestamp ImageExporterOptKey = "rewrite-timestamp"
 )

exporter/containerimage/opts.go

@@ -21,6 +21,7 @@ type ImageCommitOpts struct {
 	Epoch       *time.Time
 
 	ForceInlineAttestations bool // force inline attestations to be attached
+	RewriteTimestamp        bool // rewrite timestamps in layers to match the epoch
 }
 
 func (c *ImageCommitOpts) Load(ctx context.Context, opt map[string]string) (map[string]string, error) {
@@ -52,6 +53,8 @@ func (c *ImageCommitOpts) Load(ctx context.Context, opt map[string]string) (map[
 			err = parseBool(&c.ForceInlineAttestations, k, v)
 		case exptypes.OptKeyPreferNondistLayers:
 			err = parseBool(&c.RefCfg.PreferNonDistributable, k, v)
+		case exptypes.OptKeyRewriteTimestamp:
+			err = parseBool(&c.RewriteTimestamp, k, v)
 		default:
 			rest[k] = v
 		}

exporter/containerimage/writer.go

@@ -28,6 +28,8 @@ import (
 	attestationTypes "github.com/moby/buildkit/util/attestation"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/compression"
+	"github.com/moby/buildkit/util/contentutil"
+	"github.com/moby/buildkit/util/converter"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/purl"
 	"github.com/moby/buildkit/util/system"
@@ -134,7 +136,14 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
 
 		config := exptypes.ParseKey(inp.Metadata, exptypes.ExporterImageConfigKey, p)
 		inlineCache := exptypes.ParseKey(inp.Metadata, exptypes.ExporterInlineCache, p)
-		mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, ref, config, &remotes[0], annotations, inlineCache, opts.Epoch, session.NewGroup(sessionID))
+		remote := &remotes[0]
+		if opts.RewriteTimestamp {
+			remote, err = ic.rewriteRemoteWithEpoch(ctx, opts, remote)
+			if err != nil {
+				return nil, err
+			}
+		}
+		mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, ref, config, remote, annotations, inlineCache, opts.Epoch, session.NewGroup(sessionID))
 		if err != nil {
 			return nil, err
 		}
@@ -200,6 +209,13 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
 			}
 		}
 
+		if opts.RewriteTimestamp {
+			remote, err = ic.rewriteRemoteWithEpoch(ctx, opts, remote)
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, remote, opts.Annotations.Platform(&p.Platform), inlineCache, opts.Epoch, session.NewGroup(sessionID))
 		if err != nil {
 			return nil, err
@@ -324,6 +340,52 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC
 	return out, err
 }
 
+// rewriteImageLayerWithEpoch rewrites the file timestamps in the layer blob to match the epoch, and returns a new descriptor that points to
+// the new blob.
+//
+// If no conversion is needed, this returns nil without error.
+func rewriteImageLayerWithEpoch(ctx context.Context, cs content.Store, desc ocispecs.Descriptor, comp compression.Config, epoch *time.Time) (*ocispecs.Descriptor, error) {
+	converterFn, err := converter.NewWithRewriteTimestamp(ctx, cs, desc, comp, epoch)
+	if err != nil {
+		return nil, err
+	}
+	if converterFn == nil {
+		return nil, nil
+	}
+	return converterFn(ctx, cs, desc)
+}
+
+func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCommitOpts, remote *solver.Remote) (*solver.Remote, error) {
+	if opts.Epoch == nil {
+		bklog.G(ctx).Warn("rewrite-timestamp is specified, but no source-date-epoch was found")
+		return remote, nil
+	}
+	remoteDescriptors := remote.Descriptors
+	cs := contentutil.NewStoreWithProvider(ic.opt.ContentStore, remote.Provider)
+	eg, ctx := errgroup.WithContext(ctx)
+	rewriteDone := progress.OneOff(ctx,
+		fmt.Sprintf("rewriting layers with source-date-epoch %d (%s)", opts.Epoch.Unix(), opts.Epoch.String()))
+	for i, desc := range remoteDescriptors {
+		i, desc := i, desc
+		eg.Go(func() error {
+			if rewrittenDesc, err := rewriteImageLayerWithEpoch(ctx, cs, desc, opts.RefCfg.Compression, opts.Epoch); err != nil {
+				bklog.G(ctx).WithError(err).Warnf("failed to rewrite layer %d/%d to match source-date-epoch %d (%s)",
+					i+1, len(remoteDescriptors), opts.Epoch.Unix(), opts.Epoch.String())
+			} else if rewrittenDesc != nil {
+				remoteDescriptors[i] = *rewrittenDesc
+			}
+			return nil
+		})
+	}
+	if err := rewriteDone(eg.Wait()); err != nil {
+		return nil, err
+	}
+	return &solver.Remote{
+		Provider:    cs,
+		Descriptors: remoteDescriptors,
+	}, nil
+}
+
 func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *ImageCommitOpts, ref cache.ImmutableRef, config []byte, remote *solver.Remote, annotations *Annotations, inlineCache []byte, epoch *time.Time, sg session.Group) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) {
 	if len(config) == 0 {
 		var err error

frontend/dockerfile/dockerfile_test.go

@@ -43,12 +43,10 @@ import (
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/contentutil"
-	"github.com/moby/buildkit/util/iohelper"
 	"github.com/moby/buildkit/util/testutil"
 	"github.com/moby/buildkit/util/testutil/httpserver"
 	"github.com/moby/buildkit/util/testutil/integration"
 	"github.com/moby/buildkit/util/testutil/workers"
-	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
@@ -6549,9 +6547,16 @@ func testReproSourceDateEpoch(t *testing.T, sb integration.Sandbox) {
 	if sb.Snapshotter() == "native" {
 		t.Skip("the digest is not reproducible with the \"native\" snapshotter because hardlinks are processed in a different way: https://github.com/moby/buildkit/pull/3456#discussion_r1062650263")
 	}
+
+	registry, err := sb.NewRegistry()
+	if errors.Is(err, integration.ErrRequirements) {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+
 	f := getFrontend(t, sb)
 
-	tm := time.Date(2023, time.January, 10, 12, 34, 56, 0, time.UTC)
+	tm := time.Date(2023, time.January, 10, 12, 34, 56, 0, time.UTC) // 1673354096
 	t.Logf("SOURCE_DATE_EPOCH=%d", tm.Unix())
 
 	dockerfile := []byte(`# The base image cannot be busybox, due to https://github.com/moby/buildkit/issues/3455
@@ -6565,34 +6570,23 @@ RUN touch -d '2030-01-01 12:34:56' /foo-2030.1
 RUN rm -f /foo.1
 RUN rm -f /foo-2010.1
 RUN rm -f /foo-2030.1
-
-# Limit the timestamp upper bound to SOURCE_DATE_EPOCH.
-# Workaround for https://github.com/moby/buildkit/issues/3180
-ARG SOURCE_DATE_EPOCH
-RUN find $( ls / | grep -E -v "^(dev|mnt|proc|sys)$" ) -newermt "@${SOURCE_DATE_EPOCH}" -writable -xdev | xargs touch --date="@${SOURCE_DATE_EPOCH}" --no-dereference
-
-# Squashing is needed to apply the touched timestamps across multiple "RUN" instructions.
-# This squashing also addresses non-reproducibility of whiteout timestamps (https://github.com/moby/buildkit/issues/3168).
-FROM scratch
-COPY --from=0 / /
 `)
 
-	const expectedDigest = "sha256:d286483eccf4d57c313a3f389cdc196e668d914d319c574b15aabdf1963c5eeb"
+	const expectedDigest = "sha256:29f2980a804038b0f910af98e9ddb18bfa4d5514995ee6bb4343ddf621a4e183"
 
 	dir := integration.Tmpdir(
 		t,
 		fstest.CreateFile("Dockerfile", dockerfile, 0600),
 	)
 	defer os.RemoveAll(dir)
 
-	c, err := client.New(sb.Context(), sb.Address())
+	ctx := sb.Context()
+	c, err := client.New(ctx, sb.Address())
 	require.NoError(t, err)
 	defer c.Close()
 
-	outDigester := digest.SHA256.Digester()
-	outW := &iohelper.NopWriteCloser{Writer: outDigester.Hash()}
-
-	_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+	target := registry + "/buildkit/testreprosourcedateepoch:" + fmt.Sprintf("%d", tm.Unix())
+	solveOpt := client.SolveOpt{
 		FrontendAttrs: map[string]string{
 			"build-arg:SOURCE_DATE_EPOCH": fmt.Sprintf("%d", tm.Unix()),
 			"platform":                    "linux/amd64",
@@ -6603,17 +6597,62 @@ COPY --from=0 / /
 		},
 		Exports: []client.ExportEntry{
 			{
-				Type:   client.ExporterOCI,
-				Output: fixedWriteCloser(outW),
+				Type: client.ExporterImage,
+				Attrs: map[string]string{
+					"name":              target,
+					"push":              "true",
+					"oci-mediatypes":    "true",
+					"rewrite-timestamp": "true",
+				},
 			},
 		},
-	}, nil)
+		CacheExports: []client.CacheOptionsEntry{
+			{
+				Type: "registry",
+				Attrs: map[string]string{
+					"ref":            target + "-cache",
+					"oci-mediatypes": "true",
+					"image-manifest": "true",
+				},
+			},
+		},
+	}
+	_, err = f.Solve(ctx, c, solveOpt, nil)
 	require.NoError(t, err)
 
-	outDigest := outDigester.Digest().String()
-	t.Logf("OCI archive digest=%q", outDigest)
+	desc, manifest := readImage(t, ctx, target)
+	_, cacheManifest := readImage(t, ctx, target+"-cache")
 	t.Log("The digest may change depending on the BuildKit version, the snapshotter configuration, etc.")
-	require.Equal(t, expectedDigest, outDigest)
+	require.Equal(t, expectedDigest, desc.Digest.String())
+	// Image layers must have rewritten-timestamp
+	for _, l := range manifest.Layers {
+		require.Equal(t, fmt.Sprintf("%d", tm.Unix()), l.Annotations["buildkit/rewritten-timestamp"])
+	}
+	// Cache layers must *not* have rewritten-timestamp
+	for _, l := range cacheManifest.Layers {
+		require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
+	}
+
+	// Build again, but without rewrite-timestamp
+	solveOpt2 := solveOpt
+	delete(solveOpt2.Exports[0].Attrs, "rewrite-timestamp")
+	_, err = f.Solve(ctx, c, solveOpt2, nil)
+	require.NoError(t, err)
+	_, manifest2 := readImage(t, ctx, target)
+	for _, l := range manifest2.Layers {
+		require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
+	}
+}
+
+//nolint:revive // context-as-argument: context.Context should be the first parameter of a function
+func readImage(t *testing.T, ctx context.Context, ref string) (ocispecs.Descriptor, ocispecs.Manifest) {
+	desc, provider, err := contentutil.ProviderFromRef(ref)
+	require.NoError(t, err)
+	dt, err := content.ReadBlob(ctx, provider, desc)
+	require.NoError(t, err)
+	var manifest ocispecs.Manifest
+	require.NoError(t, json.Unmarshal(dt, &manifest))
+	return desc, manifest
 }
 
 func testNilContextInSolveGateway(t *testing.T, sb integration.Sandbox) {

util/contentutil/storewithprovider.go

@@ -0,0 +1,24 @@
+package contentutil
+
+import (
+	"context"
+
+	"github.com/containerd/containerd/content"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func NewStoreWithProvider(cs content.Store, p content.Provider) content.Store {
+	return &storeWithProvider{Store: cs, p: p}
+}
+
+type storeWithProvider struct {
+	content.Store
+	p content.Provider
+}
+
+func (cs *storeWithProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
+	if ra, err := cs.p.ReaderAt(ctx, desc); err == nil {
+		return ra, nil
+	}
+	return cs.Store.ReaderAt(ctx, desc)
+}