From 452878d665648ada0aaf816931611fdd9c683a97 Mon Sep 17 00:00:00 2001 From: Jake Sanders <1200829+dekkagaijin@users.noreply.github.com> Date: Wed, 11 Apr 2018 06:46:01 -0700 Subject: [PATCH] Allow for users to declare explicit build timestamps (#364) * Allow for users to declare explicit build timestamps. Signed-off-by: Jake Sanders * default to {BUILD_TIMESTAMP} when `stamp = True` and `creation_time` is undefined, add `creation_time` to README.md Signed-off-by: Jake Sanders * use unix epoch in seconds, instead, but assume milliseconds for values > 1e11 Signed-off-by: Jake Sanders --- README.md | 13 +++++- container/BUILD | 5 +++ container/create_image_config.py | 41 +++++++++++++++++-- container/image.bzl | 27 +++++++++---- container/image_test.py | 68 ++++++++++++++++++++++++++++++-- testdata/BUILD | 33 ++++++++++++++++ 6 files changed, 173 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cffa24a44..f5a7235e2 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,7 @@ container_push( registry = "gcr.io", repository = "my-project/my-image", tag = "{BUILD_USER}", + creation_time = "{BUILD_TIMESTAMP}", # Trigger stamping. stamp = True, @@ -1148,7 +1149,7 @@ A rule that assembles data into a tarball which can be use as in `layers` attr i ## container_image ```python -container_image(name, base, data_path, directory, files, legacy_repository_naming, mode, tars, debs, symlinks, entrypoint, cmd, env, labels, ports, volumes, workdir, layers, repository) +container_image(name, base, data_path, directory, files, legacy_repository_naming, mode, tars, debs, symlinks, entrypoint, cmd, creation_time, env, labels, ports, volumes, workdir, layers, repository) ``` @@ -1352,6 +1353,16 @@ container_image(name, base, data_path, directory, files, legacy_repository_namin

This field supports stamp variables.

+ + + +
creation_time + String, optional, default to {BUILD_TIMESTAMP} when stamp = True, otherwise 0 +

The image's creation timestamp.

+

Acceptable formats: Integer or floating point seconds since Unix + Epoch, RFC 3339 date/time.

+

This field supports stamp variables.

+
env diff --git a/container/BUILD b/container/BUILD index 6c9d4d24e..db5494575 100644 --- a/container/BUILD +++ b/container/BUILD @@ -94,6 +94,11 @@ TEST_TARGETS = [ "layers_with_docker_tarball_base", # TODO(mattmoor): Re-enable once archive is visible # "generated_tarball", + "with_unix_epoch_creation_time", + "with_millisecond_unix_epoch_creation_time", + "with_rfc_3339_creation_time", + "with_stamped_creation_time", + "with_default_stamped_creation_time", "with_env", "layers_with_env", "with_double_env", diff --git a/container/create_image_config.py b/container/create_image_config.py index b8282f85d..6bc585397 100644 --- a/container/create_image_config.py +++ b/container/create_image_config.py @@ -13,7 +13,10 @@ # limitations under the License. """This package manipulates v2.2 image configuration metadata.""" +from __future__ import division + import argparse +import datetime import json import sys @@ -40,6 +43,11 @@ parser.add_argument('--command', action='append', default=[], help='Override the "Cmd" of the previous layer.') +parser.add_argument('--creation_time', action='store', required=False, + help='The creation timestamp. Acceptable formats: ' + 'Integer or floating point seconds since Unix Epoch, RFC ' + '3339 date/time') + parser.add_argument('--user', action='store', help='The username to run commands under.') @@ -121,10 +129,37 @@ def Stamp(inp): elif '{' in value: labels[label] = Stamp(value) + creation_time = None + if args.creation_time: + creation_time = Stamp(args.creation_time) + try: + # If creation_time is parsable as a floating point type, assume unix epoch + # timestamp. + parsed_unix_timestamp = float(creation_time) + if parsed_unix_timestamp > 1.0e+11: + # Bazel < 0.12 was bugged and used milliseconds since unix epoch as + # the default. Values > 1e11 are assumed to be unix epoch + # milliseconds. + parsed_unix_timestamp = parsed_unix_timestamp / 1000.0 + + # Construct a RFC 3339 date/time from the Unix epoch. + creation_time = ( + datetime.datetime.utcfromtimestamp( + parsed_unix_timestamp + ).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + except ValueError: + # Otherwise, assume RFC 3339 date/time format. + pass + output = v2_2_metadata.Override(data, v2_2_metadata.Overrides( - author='Bazel', created_by='bazel build ...', - layers=layers, entrypoint=list(map(Stamp, fix_dashdash(args.entrypoint))), - cmd=list(map(Stamp, fix_dashdash(args.command))), user=Stamp(args.user), + author='Bazel', + created_by='bazel build ...', + layers=layers, + entrypoint=list(map(Stamp, fix_dashdash(args.entrypoint))), + cmd=list(map(Stamp, fix_dashdash(args.command))), + creation_time=creation_time, + user=Stamp(args.user), labels=labels, env={ k: Stamp(v) for (k, v) in six.iteritems(KeyValueToDict(args.env)) diff --git a/container/image.bzl b/container/image.bzl index 9077a8dec..943e04bb5 100644 --- a/container/image.bzl +++ b/container/image.bzl @@ -88,7 +88,9 @@ def _get_base_config(ctx, base): l = _get_layers(ctx, ctx.attr.base, base) return l.get("config") -def _image_config(ctx, layer_names, entrypoint=None, cmd=None, env=None, base_config=None, layer_name=None): +def _image_config(ctx, layer_names, entrypoint=None, cmd=None, + creation_time=None, env=None, base_config=None, + layer_name=None): """Create the configuration for a new container image.""" config = ctx.new_file(ctx.label.name + "." + layer_name + ".config") @@ -114,6 +116,13 @@ def _image_config(ctx, layer_names, entrypoint=None, cmd=None, env=None, base_co ] + [ "--volumes=%s" % x for x in ctx.attr.volumes ] + if creation_time: + args += ["--creation_time=%s" % creation_time] + elif ctx.attr.stamp: + # If stamping is enabled, and the creation_time is not manually defined, + # default to '{BUILD_TIMESTAMP}'. + args += ["--creation_time={BUILD_TIMESTAMP}"] + _labels = _serialize_dict(labels) if _labels: args += ["--labels=%s" % x for x in _labels.split(',')] @@ -162,8 +171,9 @@ def _repository_name(ctx): def _impl(ctx, base=None, files=None, file_map=None, empty_files=None, empty_dirs=None, directory=None, entrypoint=None, cmd=None, - symlinks=None, env=None, layers=None, debs=None, tars=None, - output_executable=None, output_tarball=None, output_layer=None): + creation_time=None, symlinks=None, env=None, layers=None, debs=None, + tars=None, output_executable=None, output_tarball=None, + output_layer=None): """Implementation for the container_image rule. Args: @@ -176,6 +186,7 @@ def _impl(ctx, base=None, files=None, file_map=None, empty_files=None, directory: str, overrides ctx.attr.directory entrypoint: str List, overrides ctx.attr.entrypoint cmd: str List, overrides ctx.attr.cmd + creation_time: str, overrides ctx.attr.creation_time symlinks: str Dict, overrides ctx.attr.symlinks env: str Dict, overrides ctx.attr.env layers: label List, overrides ctx.attr.layers @@ -185,8 +196,9 @@ def _impl(ctx, base=None, files=None, file_map=None, empty_files=None, output_tarball: File, overrides ctx.outputs.out output_layer: File, overrides ctx.outputs.layer """ - entrypoint = entrypoint or ctx.attr.entrypoint - cmd = cmd or ctx.attr.cmd + entrypoint=entrypoint or ctx.attr.entrypoint + cmd=cmd or ctx.attr.cmd + creation_time=creation_time or ctx.attr.creation_time output_executable = output_executable or ctx.outputs.executable output_tarball = output_tarball or ctx.outputs.out output_layer = output_layer or ctx.outputs.layer @@ -220,8 +232,8 @@ def _impl(ctx, base=None, files=None, file_map=None, empty_files=None, for i, layer in enumerate(layers): config_file, config_digest = _image_config( ctx, [layer_diff_ids[i]], - entrypoint=entrypoint, cmd=cmd, env=layer.env, - base_config=config_file, layer_name=str(i), ) + entrypoint=entrypoint, cmd=cmd, creation_time=creation_time, + env=layer.env, base_config=config_file, layer_name=str(i), ) # Construct a temporary name based on the build target. tag_name = _repository_name(ctx) + ":" + ctx.label.name @@ -280,6 +292,7 @@ _attrs = dict(_layer.attrs.items() + { "user": attr.string(), "labels": attr.string_dict(), "cmd": attr.string_list(), + "creation_time": attr.string(), "entrypoint": attr.string_list(), "ports": attr.string_list(), # Skylark doesn't support int_list... "volumes": attr.string_list(), diff --git a/container/image_test.py b/container/image_test.py index 06793ec1e..eab47a6d5 100644 --- a/container/image_test.py +++ b/container/image_test.py @@ -13,6 +13,7 @@ # limitations under the License. import cStringIO +import datetime import json import os import tarfile @@ -180,6 +181,67 @@ def test_derivative_with_volume(self): '/asdf': {}, '/blah': {}, '/logs': {} }) + def test_with_unix_epoch_creation_time(self): + with TestImage('with_unix_epoch_creation_time') as img: + self.assertDigest(img, '85113de3854559f724a23eed6afea5ceecd5fd4bf241cedaded8af0474d4f882') + self.assertEqual(2, len(img.fs_layers())) + cfg = json.loads(img.config_file()) + self.assertEqual('2009-02-13T23:31:30.120000Z', cfg.get('created', '')) + + def test_with_millisecond_unix_epoch_creation_time(self): + with TestImage('with_millisecond_unix_epoch_creation_time') as img: + self.assertDigest(img, 'e9412cb69da02e05fd5b7f8cc1a5d60139c091362afdc2488f9c8f7c508e5d3b') + self.assertEqual(2, len(img.fs_layers())) + cfg = json.loads(img.config_file()) + self.assertEqual('2009-02-13T23:31:30.123450Z', cfg.get('created', '')) + + def test_with_rfc_3339_creation_time(self): + with TestImage('with_rfc_3339_creation_time') as img: + self.assertDigest(img, '9aeef8cba32f3af6e95a08e60d76cc5e2a46de4847da5366bffeb1b3d7066d17') + self.assertEqual(2, len(img.fs_layers())) + cfg = json.loads(img.config_file()) + self.assertEqual('1989-05-03T12:58:12.345Z', cfg.get('created', '')) + + def test_with_stamped_creation_time(self): + with TestImage('with_stamped_creation_time') as img: + self.assertEqual(2, len(img.fs_layers())) + cfg = json.loads(img.config_file()) + created_str = cfg.get('created', '') + self.assertNotEqual('', created_str) + + now = datetime.datetime.utcnow() + + created = datetime.datetime.strptime(created_str, '%Y-%m-%dT%H:%M:%S.%fZ') + + # The BUILD_TIMESTAMP is set by Bazel to Java's CurrentTimeMillis / 1000, + # or env['SOURCE_DATE_EPOCH']. For Bazel versions before 0.12, there was + # a bug where CurrentTimeMillis was not divided by 1000. + # See https://github.com/bazelbuild/bazel/issues/2240 + # https://bazel-review.googlesource.com/c/bazel/+/48211 + # Assume that any value for 'created' within a reasonable bound is fine. + self.assertLessEqual(now - created, datetime.timedelta(minutes=5)) + + def test_with_default_stamped_creation_time(self): + # {BUILD_TIMESTAMP} should be the default when `stamp = True` and + # `creation_time` isn't explicitly defined. + with TestImage('with_default_stamped_creation_time') as img: + self.assertEqual(2, len(img.fs_layers())) + cfg = json.loads(img.config_file()) + created_str = cfg.get('created', '') + self.assertNotEqual('', created_str) + + now = datetime.datetime.utcnow() + + created = datetime.datetime.strptime(created_str, '%Y-%m-%dT%H:%M:%S.%fZ') + + # The BUILD_TIMESTAMP is set by Bazel to Java's CurrentTimeMillis / 1000, + # or env['SOURCE_DATE_EPOCH']. For Bazel versions before 0.12, there was + # a bug where CurrentTimeMillis was not divided by 1000. + # See https://github.com/bazelbuild/bazel/issues/2240 + # https://bazel-review.googlesource.com/c/bazel/+/48211 + # Assume that any value for 'created' within a reasonable bound is fine. + self.assertLessEqual(now - created, datetime.timedelta(minutes=5)) + def test_with_env(self): with TestBundleImage( 'with_env', 'bazel/%s:with_env' % TEST_DATA_TARGET_BASE) as img: @@ -509,7 +571,7 @@ def test_py3_image_args(self): '/app/testdata/py3_image.binary', 'arg0', 'arg1']) - + def test_java_image_args(self): with TestImage('java_image') as img: self.assertConfigEqual(img, 'Entrypoint', [ @@ -537,7 +599,7 @@ def test_go_image_args(self): '/app/testdata/rust_image_binary', 'arg0', 'arg1']) - + def test_scala_image_args(self): with TestImage('scala_image') as img: self.assertConfigEqual(img, 'Entrypoint', [ @@ -576,7 +638,7 @@ def test_nodejs_image_args(self): '/app/testdata/nodejs_image.binary', 'arg0', 'arg1']) - + if __name__ == '__main__': unittest.main() diff --git a/testdata/BUILD b/testdata/BUILD index 67e52133d..d04c47342 100644 --- a/testdata/BUILD +++ b/testdata/BUILD @@ -244,6 +244,39 @@ container_image( # ], # ) +container_image( + name = "with_unix_epoch_creation_time", + base = ":base_with_volume", + creation_time = "1234567890.12", +) + +# This is to support bazel < 0.12 BUILD_TIMESTAMPs, which were in milliseconds +# by default. +container_image( + name = "with_millisecond_unix_epoch_creation_time", + base = ":base_with_volume", + creation_time = "1234567890123.45", +) + +container_image( + name = "with_rfc_3339_creation_time", + base = ":base_with_volume", + creation_time = "1989-05-03T12:58:12.345Z", +) + +container_image( + name = "with_stamped_creation_time", + base = ":base_with_volume", + creation_time = "{BUILD_TIMESTAMP}", + stamp = True, +) + +container_image( + name = "with_default_stamped_creation_time", + base = ":base_with_volume", + stamp = True, +) + container_image( name = "with_env", base = ":base_with_volume",