diff --git a/README.md b/README.md index cac512c6..7a1533c6 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,12 @@ A list of images to attempt pulling before building in the format `service:CACHE They will be mapped directly to `cache-from` elements in the build according to the spec so any valid format there should be allowed. +#### `cache-to` (build only, string or array) + +A list of export locations to be used to share build cache with future builds in the format `service:CACHE-SPEC` to allow for layer re-use. Unsupported caches are ignored and do not prevent building images. + +They will be mapped directly to `cache-to` elements in the build according to the spec so any valid format there should be allowed. + #### `target` (build only) Allow for intermediate builds as if building with docker's `--target VALUE` options. @@ -343,7 +349,7 @@ The default is `false`. To run the tests: ```bash -docker-compose run --rm tests bats tests tests/v1 +docker compose run --rm tests bats tests tests/v1 ``` ## License diff --git a/commands/build.sh b/commands/build.sh index 7d9d67c5..49867642 100755 --- a/commands/build.sh +++ b/commands/build.sh @@ -35,8 +35,24 @@ get_caches_for_service() { fi } +get_caches_to_service() { + local service="$1" + + # Read any cache-to parameters provided + for line in $(plugin_read_list CACHE_TO) ; do + IFS=':' read -r -a tokens <<< "$line" + service_name=${tokens[0]} + service_image=$(IFS=':'; echo "${tokens[*]:1}") + + if [ "${service_name}" == "${service}" ]; then + echo "$service_image" + fi + done +} + + # Run through all images in the build property, either a single item or a list -# and build up a list of service name, image name and optional cache-froms to +# and build up a list of service name, image name and optional cache-froms and cache-tos to # write into a docker-compose override file for service_name in $(plugin_read_list BUILD) ; do target="$(plugin_read_config TARGET "")" @@ -47,19 +63,29 @@ for service_name in $(plugin_read_list BUILD) ; do cache_from+=("$cache_line") done + cache_to=() + for cache_line in $(get_caches_to_service "$service_name"); do + cache_to+=("$cache_line") + done + labels=() while read -r label ; do [[ -n "${label:-}" ]] && labels+=("${label}") done <<< "$(plugin_read_list BUILD_LABELS)" - if [[ -n "${target}" ]] || [[ "${#labels[@]}" -gt 0 ]] || [[ "${#cache_from[@]}" -gt 0 ]]; then + if [[ -n "${target}" ]] || [[ "${#labels[@]}" -gt 0 ]] || [[ "${#cache_to[@]}" -gt 0 ]] || [[ "${#cache_from[@]}" -gt 0 ]]; then build_images+=("$service_name" "${image_name}" "${target}") - + build_images+=("${#cache_from[@]}") if [[ "${#cache_from[@]}" -gt 0 ]]; then build_images+=("${cache_from[@]}") fi - + + build_images+=("${#cache_to[@]}") + if [[ "${#cache_to[@]}" -gt 0 ]]; then + build_images+=("${cache_to[@]}") + fi + build_images+=("${#labels[@]}") if [[ "${#labels[@]}" -gt 0 ]]; then build_images+=("${labels[@]}") diff --git a/commands/run.sh b/commands/run.sh index e1b493f6..b3de229b 100755 --- a/commands/run.sh +++ b/commands/run.sh @@ -52,7 +52,7 @@ for service_name in "${prebuilt_candidates[@]}" ; do fi if [[ -n "$prebuilt_image" ]] ; then - prebuilt_service_overrides+=("$service_name" "$prebuilt_image" "" 0 0) + prebuilt_service_overrides+=("$service_name" "$prebuilt_image" "" 0 0 0) prebuilt_services+=("$service_name") # If it's prebuilt, we need to pull it down diff --git a/lib/shared.bash b/lib/shared.bash index 53ca4956..36155cb5 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -133,11 +133,15 @@ function build_image_override_file() { "$(docker_compose_config_version)" "$@" } -# Checks that a specific version of docker-compose supports cache_from -function docker_compose_supports_cache_from() { +# Checks that a specific version of docker-compose supports cache_from and cache_to +function docker_compose_supports_cache() { local version="$1" if [[ "$version" == 1* || "$version" =~ ^(2|3)(\.[01])?$ ]] ; then - return 1 + echo "Unsupported Docker Compose config file version: $version" + echo "The 'cache_from' option can only be used with Compose file versions 2.2 or 3.2 and above." + echo "For more information on Docker Compose configuration file versions, see:" + echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning" + exit 1 fi } @@ -176,6 +180,16 @@ function build_image_override_file_with_version() { done fi + # load cache_to array + cache_to_amt="${1:-0}" + [[ -n "${1:-}" ]] && shift; # remove the value if not empty + if [[ "${cache_to_amt}" -gt 0 ]]; then + cache_to=() + for _ in $(seq 1 "$cache_to_amt"); do + cache_to+=( "$1" ); shift + done + fi + # load labels array labels_amt="${1:-0}" [[ -n "${1:-}" ]] && shift; # remove the value if not empty @@ -186,7 +200,7 @@ function build_image_override_file_with_version() { done fi - if [[ -z "$image_name" ]] && [[ -z "$target" ]] && [[ "$cache_from_amt" -eq 0 ]] && [[ "$labels_amt" -eq 0 ]]; then + if [[ -z "$image_name" ]] && [[ -z "$target" ]] && [[ "$cache_from_amt" -eq 0 ]] && [[ "$cache_to_amt" -eq 0 ]] && [[ "$labels_amt" -eq 0 ]]; then # should not print out an empty service continue fi @@ -197,7 +211,7 @@ function build_image_override_file_with_version() { printf " image: %s\\n" "$image_name" fi - if [[ "$cache_from_amt" -gt 0 ]] || [[ -n "$target" ]] || [[ "$labels_amt" -gt 0 ]]; then + if [[ "$cache_from_amt" -gt 0 ]] || [[ "$cache_to_amt" -gt 0 ]] || [[ -n "$target" ]] || [[ "$labels_amt" -gt 0 ]]; then printf " build:\\n" fi @@ -206,13 +220,7 @@ function build_image_override_file_with_version() { fi if [[ "$cache_from_amt" -gt 0 ]] ; then - if ! docker_compose_supports_cache_from "$version" ; then - echo "Unsupported Docker Compose config file version: $version" - echo "The 'cache_from' option can only be used with Compose file versions 2.2 or 3.2 and above." - echo "For more information on Docker Compose configuration file versions, see:" - echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning" - exit 1 - fi + docker_compose_supports_cache "$version" printf " cache_from:\\n" for cache_from_i in "${cache_from[@]}"; do @@ -220,6 +228,15 @@ function build_image_override_file_with_version() { done fi + if [[ "$cache_to_amt" -gt 0 ]] ; then + docker_compose_supports_cache "$version" + + printf " cache_to:\\n" + for cache_to_i in "${cache_to[@]}"; do + printf " - %s\\n" "${cache_to_i}" + done + fi + if [[ "$labels_amt" -gt 0 ]] ; then printf " labels:\\n" for label in "${labels[@]}"; do diff --git a/plugin.yml b/plugin.yml index a1c8f370..4e269e5a 100755 --- a/plugin.yml +++ b/plugin.yml @@ -9,10 +9,10 @@ configuration: run: type: string build: - type: [ string, array ] + type: [string, array] minimum: 1 push: - type: [ string, array ] + type: [string, array] minimum: 1 ansi: @@ -35,6 +35,9 @@ configuration: cache-from: type: [ string, array ] minimum: 1 + cache-to: + type: [ string, array ] + minimum: 1 cli-version: oneOf: - type: string @@ -150,6 +153,7 @@ configuration: buildkit: [ build ] buildkit-inline-cache: [ build ] cache-from: [ build ] + cache-to: [ build ] command: [ run ] dependencies: [ run ] entrypoint: [ run ] diff --git a/tests/docker-compose-config.bats b/tests/docker-compose-config.bats index 910d2531..ff56a48c 100644 --- a/tests/docker-compose-config.bats +++ b/tests/docker-compose-config.bats @@ -81,34 +81,34 @@ load '../lib/shared' assert_output "2.0" } -@test "Whether docker-compose supports cache_from directive" { - run docker_compose_supports_cache_from "" +@test "Whether docker-compose supports cache_from and cache_to directive" { + run docker_compose_supports_cache "" assert_success - run docker_compose_supports_cache_from "1.0" + run docker_compose_supports_cache "1.0" assert_failure - run docker_compose_supports_cache_from "2" + run docker_compose_supports_cache "2" assert_failure - run docker_compose_supports_cache_from "2.1" + run docker_compose_supports_cache "2.1" assert_failure - run docker_compose_supports_cache_from "2.2" + run docker_compose_supports_cache "2.2" assert_success - run docker_compose_supports_cache_from "2.3" + run docker_compose_supports_cache "2.3" assert_success - run docker_compose_supports_cache_from "3" + run docker_compose_supports_cache "3" assert_failure - run docker_compose_supports_cache_from "3.1" + run docker_compose_supports_cache "3.1" assert_failure - run docker_compose_supports_cache_from "3.2" + run docker_compose_supports_cache "3.2" assert_success - run docker_compose_supports_cache_from "3.3" + run docker_compose_supports_cache "3.3" assert_success } diff --git a/tests/image-override-file.bats b/tests/image-override-file.bats index 94d20301..9f4b680f 100644 --- a/tests/image-override-file.bats +++ b/tests/image-override-file.bats @@ -29,8 +29,8 @@ EOF ) run build_image_override_file_with_version "2.1" \ - "myservice1" "newimage1:1.0.0" "" 0 0 \ - "myservice2" "newimage2:1.0.0" "" 0 0 + "myservice1" "newimage1:1.0.0" "" 0 0 0 \ + "myservice2" "newimage2:1.0.0" "" 0 0 0 assert_success assert_output "$myservice_override_file2" @@ -102,7 +102,7 @@ services: EOF ) - run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 0 1 "com.buildkite.test=test" + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 0 0 1 "com.buildkite.test=test" assert_success assert_output "$myservice_override_file3" @@ -121,7 +121,7 @@ services: EOF ) - run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 0 2 "com.buildkite.test=test" "com.buildkite.test2=test2" + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 0 0 2 "com.buildkite.test=test" "com.buildkite.test2=test2" assert_success assert_output "$myservice_override_file3" @@ -143,7 +143,7 @@ services: EOF ) - run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 2 "my.repository/myservice:latest" "my.repository/myservice:target" 2 "com.buildkite.test=test" "com.buildkite.test2=test2" + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 2 "my.repository/myservice:latest" "my.repository/myservice:target" 0 2 "com.buildkite.test=test" "com.buildkite.test2=test2" assert_success assert_output "$myservice_override_file3" @@ -166,7 +166,7 @@ services: EOF ) - run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "build" 2 "my.repository/myservice:latest" "my.repository/myservice:target" 2 "com.buildkite.test=test" "com.buildkite.test2=test2" + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "build" 2 "my.repository/myservice:latest" "my.repository/myservice:target" 0 2 "com.buildkite.test=test" "com.buildkite.test2=test2" assert_success assert_output "$myservice_override_file3" @@ -207,3 +207,126 @@ EOF assert_failure } + +@test "Build a docker-compose file with cache-to" { + myservice_override_file3=$(cat <<-EOF +version: '3.2' +services: + myservice: + image: newimage:1.0.0 + build: + cache_to: + - user/app:cache +EOF + ) + + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 0 1 "user/app:cache" + + assert_success + assert_output "$myservice_override_file3" +} + +@test "Build a docker-compose file with multiple cache-to entries" { + myservice_override_file4=$(cat <<-EOF +version: '3.2' +services: + myservice: + image: newimage:1.0.0 + build: + cache_to: + - user/app:cache + - type=local,dest=path/to/cache +EOF + ) + + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 0 2 "user/app:cache" "type=local,dest=path/to/cache" + + assert_success + assert_output "$myservice_override_file4" +} + + +@test "Build a docker-compose file with cache-from and cache-to" { + myservice_override_file3=$(cat <<-EOF +version: '3.2' +services: + myservice: + image: newimage:1.0.0 + build: + cache_from: + - my.repository/myservice:latest + cache_to: + - user/app:cache +EOF + ) + + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "" 1 "my.repository/myservice:latest" 1 "user/app:cache" + + assert_success + assert_output "$myservice_override_file3" +} + +@test "Build a docker-compose file with multiple cache-from, multiple cache-to and multiple labels and target" { + myservice_override_file3=$(cat <<-EOF +version: '3.2' +services: + myservice: + image: newimage:1.0.0 + build: + target: build + cache_from: + - my.repository/myservice:latest + - my.repository/myservice:target + cache_to: + - user/app:cache + - type=local,dest=path/to/cache + labels: + - com.buildkite.test=test + - com.buildkite.test2=test2 +EOF + ) + + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "build" 2 "my.repository/myservice:latest" "my.repository/myservice:target" 2 "user/app:cache" "type=local,dest=path/to/cache" 2 "com.buildkite.test=test" "com.buildkite.test2=test2" + + assert_success + assert_output "$myservice_override_file3" +} + +@test "Build a docker-compose file with multiple services, multiple cache-from, multiple cache-to and multiple labels and target" { + myservice_override_file3=$(cat <<-EOF +version: '3.2' +services: + myservice-1: + image: newimage:1.0.0 + build: + target: build + cache_from: + - my.repository/myservice-1:latest + - my.repository/myservice-1:target + cache_to: + - user/app:cache + - type=local,dest=path/to/cache + labels: + - com.buildkite.test=test + - com.buildkite.test2=test2 + myservice-2: + image: newimage:2.0.0 + build: + target: build-2 + cache_from: + - my.repository/myservice-2:latest + - my.repository/myservice-2:target + cache_to: + - user/app:cache + - type=local,dest=path/to/cache-2 + labels: + - com.buildkite.test3=test3 + - com.buildkite.test4=test4 +EOF + ) + + run build_image_override_file_with_version "3.2" "myservice-1" "newimage:1.0.0" "build" 2 "my.repository/myservice-1:latest" "my.repository/myservice-1:target" 2 "user/app:cache" "type=local,dest=path/to/cache" 2 "com.buildkite.test=test" "com.buildkite.test2=test2" "myservice-2" "newimage:2.0.0" "build-2" 2 "my.repository/myservice-2:latest" "my.repository/myservice-2:target" 2 "user/app:cache" "type=local,dest=path/to/cache-2" 2 "com.buildkite.test3=test3" "com.buildkite.test4=test4" + + assert_success + assert_output "$myservice_override_file3" +}