commit: add --source-date-epoch and --rewrite-timestamp flags

Add a --source-date-epoch flag, defaulting to $SOURCE_DATE_EPOCH if set,
which sets the created-on date and the timestamp for the new history
entries, but does not default to modifying the timestamps on contents in
new layers.

Add a --rewrite-timestamp flag, which "clamps" timestamps in the new
layers to not be later than the --source-date-epoch value if both
the --rewrite-timestamp and --source-date-epoch flags were set.

Signed-off-by: Nalin Dahyabhai <[email protected]>

Author: nalind

cmd/buildah/commit.go

@@ -5,11 +5,13 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/containers/buildah"
 	"github.com/containers/buildah/define"
+	"github.com/containers/buildah/internal"
 	"github.com/containers/buildah/pkg/cli"
 	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/buildah/util"
@@ -39,6 +41,8 @@ type commitInputOptions struct {
 	manifest           string
 	omitTimestamp      bool
 	timestamp          int64
+	sourceDateEpoch    string
+	rewriteTimestamp   bool
 	quiet              bool
 	referenceTime      string
 	rm                 bool
@@ -117,7 +121,14 @@ func commitListFlagSet(cmd *cobra.Command, opts *commitInputOptions) {
 	flags.StringVar(&opts.iidfile, "iidfile", "", "write the image ID to the file")
 	_ = cmd.RegisterFlagCompletionFunc("iidfile", completion.AutocompleteDefault)
 	flags.BoolVar(&opts.omitTimestamp, "omit-timestamp", false, "set created timestamp to epoch 0 to allow for deterministic builds")
-	flags.Int64Var(&opts.timestamp, "timestamp", 0, "set created timestamp to epoch seconds to allow for deterministic builds, defaults to current time")
+	sourceDateEpochUsageDefault := "current time"
+	if v := os.Getenv(internal.SourceDateEpochName); v != "" {
+		sourceDateEpochUsageDefault = fmt.Sprintf("%q", v)
+	}
+	flags.StringVar(&opts.sourceDateEpoch, "source-date-epoch", os.Getenv(internal.SourceDateEpochName), "set new timestamps in image info to `seconds` after the epoch, defaults to "+sourceDateEpochUsageDefault)
+	_ = cmd.RegisterFlagCompletionFunc("source-date-epoch", completion.AutocompleteNone)
+	flags.BoolVar(&opts.rewriteTimestamp, "rewrite-timestamp", false, "set timestamps in layer to no later than the value for --source-date-epoch")
+	flags.Int64Var(&opts.timestamp, "timestamp", 0, "set new timestamps in image info and layer to `seconds` after the epoch, defaults to current times")
 	_ = cmd.RegisterFlagCompletionFunc("timestamp", completion.AutocompleteNone)
 	flags.BoolVarP(&opts.quiet, "quiet", "q", false, "don't output progress information when writing images")
 	flags.StringVar(&opts.referenceTime, "reference-time", "", "set the timestamp on the image to match the named `file`")
@@ -317,6 +328,16 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 		timestamp := finfo.ModTime().UTC()
 		options.HistoryTimestamp = &timestamp
 	}
+	if iopts.sourceDateEpoch != "" {
+		exclusiveFlags++
+		sourceDateEpochVal, err := strconv.ParseInt(iopts.sourceDateEpoch, 10, 64)
+		if err != nil {
+			return fmt.Errorf("parsing source date epoch %q: %w", iopts.sourceDateEpoch, err)
+		}
+		sourceDateEpoch := time.Unix(sourceDateEpochVal, 0).UTC()
+		options.SourceDateEpoch = &sourceDateEpoch
+	}
+	options.RewriteTimestamp = iopts.rewriteTimestamp
 	if c.Flag("timestamp").Changed {
 		exclusiveFlags++
 		timestamp := time.Unix(iopts.timestamp, 0).UTC()
@@ -327,6 +348,9 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 		timestamp := time.Unix(0, 0).UTC()
 		options.HistoryTimestamp = &timestamp
 	}
+	if exclusiveFlags > 1 {
+		return errors.New("cannot use more then one timestamp option at at time")
+	}
 
 	if iopts.cwOptions != "" {
 		confidentialWorkloadOptions, err := parse.GetConfidentialWorkloadOptions(iopts.cwOptions)
@@ -352,10 +376,6 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 		options.SBOMScanOptions = sbomOptions
 	}
 
-	if exclusiveFlags > 1 {
-		return errors.New("can not use more then one timestamp option at at time")
-	}
-
 	if !iopts.quiet {
 		options.ReportWriter = os.Stderr
 	}

commit.go

@@ -58,9 +58,20 @@ type CommitOptions struct {
 	// ReportWriter is an io.Writer which will be used to log the writing
 	// of the new image.
 	ReportWriter io.Writer
-	// HistoryTimestamp is the timestamp used when creating new items in the
-	// image's history.  If unset, the current time will be used.
+	// HistoryTimestamp specifies a timestamp to use for the image's
+	// created-on date, the corresponding field in new history entries, and
+	// the timestamps to set on contents in new layer diffs.  If left
+	// unset, the current time is used for the configuration and manifest,
+	// and timestamps of layer contents are used as-is.
 	HistoryTimestamp *time.Time
+	// SourceDateEpoch specifies a timestamp to use for the image's
+	// created-on date and the corresponding field in new history entries.
+	// If left unset, the current time is used for the configuration and
+	// manifest.
+	SourceDateEpoch *time.Time
+	// RewriteTimestamp, if set, forces timestamps in generated layers to
+	// not be later than the SourceDateEpoch, if it is set.
+	RewriteTimestamp bool
 	// github.com/containers/image/types SystemContext to hold credentials
 	// and other authentication/authorization information.
 	SystemContext *types.SystemContext
@@ -274,8 +285,9 @@ func (b *Builder) addManifest(ctx context.Context, manifestName string, imageSpe
 // if commit was successful and the image destination was local.
 func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options CommitOptions) (string, reference.Canonical, digest.Digest, error) {
 	var (
-		imgID string
-		src   types.ImageReference
+		imgID                string
+		src                  types.ImageReference
+		destinationTimestamp *time.Time
 	)
 
 	// If we weren't given a name, build a destination reference using a
@@ -293,6 +305,10 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 		timestamp := time.Unix(0, 0).UTC()
 		options.HistoryTimestamp = &timestamp
 	}
+	destinationTimestamp = options.HistoryTimestamp
+	if options.SourceDateEpoch != nil {
+		destinationTimestamp = options.SourceDateEpoch
+	}
 	nameToRemove := ""
 	if dest == nil {
 		nameToRemove = stringid.GenerateRandomID() + "-tmp"
@@ -415,7 +431,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	}
 
 	var manifestBytes []byte
-	if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy, options.OciEncryptLayers, options.OciEncryptConfig, nil, options.HistoryTimestamp), options.MaxRetries, options.RetryDelay); err != nil {
+	if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy, options.OciEncryptLayers, options.OciEncryptConfig, nil, destinationTimestamp), options.MaxRetries, options.RetryDelay); err != nil {
 		return imgID, nil, "", fmt.Errorf("copying layers and metadata for container %q: %w", b.ContainerID, err)
 	}
 	// If we've got more names to attach, and we know how to do that for

commit_test.go

@@ -174,9 +174,9 @@ func TestCommitLinkedLayers(t *testing.T) {
 	}
 	b.AddAppendedLinkedLayer(nil, imageName(layerNumber+6), "", "", ninthArchiveFile)
 	ref, err = imageStorage.Transport.ParseStoreReference(store, imageName(layerNumber))
-	require.NoError(t, err, "parsing reference for to-be-committed image", imageName(layerNumber))
+	require.NoErrorf(t, err, "parsing reference for to-be-committed image %q", imageName(layerNumber))
 	_, _, _, err = b.Commit(ctx, ref, commitOptions)
-	require.NoError(t, err, "committing", imageName(layerNumber))
+	require.NoErrorf(t, err, "committing %q", imageName(layerNumber))
 
 	// Build one last image based on the previous one.
 	builderOptions.FromImage = imageName(layerNumber)

digester.go

@@ -133,7 +133,7 @@ func newTarFilterer(writeCloser io.WriteCloser, filter func(hdr *tar.Header) (sk
 				}
 				hdr, err = tarReader.Next()
 			}
-			if err != io.EOF {
+			if !errors.Is(err, io.EOF) {
 				filterer.err = fmt.Errorf("reading tar archive: %w", err)
 				break
 			}
@@ -146,7 +146,11 @@ func newTarFilterer(writeCloser io.WriteCloser, filter func(hdr *tar.Header) (sk
 		if err == nil {
 			err = err1
 		}
-		pipeReader.CloseWithError(err)
+		if err != nil {
+			pipeReader.CloseWithError(err)
+		} else {
+			pipeReader.Close()
+		}
 		filterer.wg.Done()
 	}()
 	return filterer

docs/buildah-commit.1.md

@@ -26,8 +26,8 @@ The image ID of the image that was created.  On error, 1 is returned and errno i
 Read the contents of the file `source` and add it to the committed image as a
 file at `destination`.  If `destination` is not specified, the path of `source`
 will be used.  The new file will be owned by UID 0, GID 0, have 0644
-permissions, and be given a current timestamp unless the **--timestamp** option
-is also specified.  This option can be specified multiple times.
+permissions, and be given the timestamp specified to the **--timestamp** option
+if it is specified.  This option can be specified multiple times.
 
 **--authfile** *path*
 
@@ -196,6 +196,13 @@ the image is not present locally.
 
 When writing the output image, suppress progress output.
 
+**--rewrite-timestamp**
+
+When generating the new layer for the image, ensure that no newly added content
+bears a timestamp later than the value used by the **--source-date-epoch**
+flag, if one was provided, by replacing any timestamps which are later than
+that value, with that value.
+
 **--rm**
 Remove the working container and its contents after creating the image.
 Default leaves the container and its content in place.
@@ -295,16 +302,41 @@ Generate SBOMs using the specified scanner image.
 
 Sign the new image using the GPG key that matches the specified fingerprint.
 
+**--source-date-epoch** *seconds*
+
+Set the "created" timestamp for the image to this number of seconds since the
+epoch (Unix time 0, i.e., 00:00:00 UTC on 1 January 1970) to make it easier to
+create deterministic builds (defaults to $SOURCE_DATE_EPOCH if set, otherwise
+the current time will be used).
+
+The "created" timestamp is written into the image's configuration and manifest
+when the image is committed, so committing the same working container at two
+different times will produce images with different sha256 hashes, even if no
+other changes were made to the working container in between.
+
+When --source-date-epoch is set, the "created" timestamp is always set to the time
+specified, which should allow for identical images to be committed at different
+times.
+
 **--squash**
 
 Squash all of the new image's layers (including those inherited from a base image) into a single new layer.
 
 **--timestamp** *seconds*
 
-Set the create timestamp to seconds since epoch to allow for deterministic builds (defaults to current time).
-By default, the created timestamp is changed and written into the image manifest with every commit,
-causing the image's sha256 hash to be different even if the sources are exactly the same otherwise.
-When --timestamp is set, the created timestamp is always set to the time specified and therefore not changed, allowing the image's sha256 to remain the same. All files committed to the layers of the image will be created with the timestamp.
+Set the "created" timestamp for the image to this number of seconds since the
+epoch (Unix time 0, i.e., 00:00:00 UTC on 1 January 1970) to make it easier to
+create deterministic builds (defaults to current time).
+
+The "created" timestamp is written into the image's configuration and manifest
+when the image is committed, so committing the same working container at two
+different times will produce images with different sha256 hashes, even if no
+other changes were made to the working container in between.
+
+When --timestamp is set, the "created" timestamp is always set to the time
+specified, which should allow for identical images to be committed at different
+times.  All content in the new layer added as part of the image will also bear
+this timestamp.
 
 **--tls-verify** *bool-value*