feat: reuse cached layers to reduce build times

.github/workflows/build-cpu.yml

@@ -70,6 +70,7 @@ jobs:
                                   --image_name ${{ env.IMAGE_NAME }} \
                                   --tag_pinned ${{ github.sha }} \
                                   --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                  --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                   --base_container $UPSTREAM_CONTAINER_CPU \
                                   --push
         )
@@ -96,6 +97,7 @@ jobs:
                                  --image_name ${{ env.IMAGE_NAME }} \
                                  --tag_pinned ${{ github.sha }} \
                                  --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                 --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                  --base_container ${{ steps.base-notebook-cpu.outputs.tag_pinned }} \
                                  --push
 
@@ -121,6 +123,7 @@ jobs:
                                  --image_name ${{ env.IMAGE_NAME }} \
                                  --tag_pinned ${{ github.sha }} \
                                  --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                 --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                  --base_container ${{ steps.minimal-notebook-cpu.outputs.tag_pinned }} \
                                  --push
 
@@ -146,6 +149,7 @@ jobs:
                                  --image_name ${{ env.IMAGE_NAME }} \
                                  --tag_pinned ${{ github.sha }} \
                                  --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                 --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                  --base_container ${{ steps.minimal-notebook-cpu.outputs.tag_pinned }} \
                                  --push
 
@@ -171,6 +175,7 @@ jobs:
                                  --image_name ${{ env.IMAGE_NAME }} \
                                  --tag_pinned ${{ github.sha }} \
                                  --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                 --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                  --base_container ${{ steps.minimal-notebook-cpu.outputs.tag_pinned }} \
                                  --push
 

.github/workflows/build-gpu.yml

@@ -95,6 +95,7 @@ jobs:
                                  --image_name ${{ env.IMAGE_NAME }} \
                                  --tag_pinned ${{ github.sha }} \
                                  --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                 --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                  --base_container ${{ steps.upstream-equivalent-notebook-gpu.outputs.tag_pinned }} \
                                  --push
 
@@ -120,6 +121,7 @@ jobs:
                                  --image_name ${{ env.IMAGE_NAME }} \
                                  --tag_pinned ${{ github.sha }} \
                                  --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                 --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                  --base_container ${{ steps.base-notebook-gpu.outputs.tag_pinned }} \
                                  --push
 
@@ -145,6 +147,7 @@ jobs:
                                  --image_name ${{ env.IMAGE_NAME }} \
                                  --tag_pinned ${{ github.sha }} \
                                  --tag_latest ${{ steps.get_branch_name.outputs.branch_name }} \
+                                 --cache_from ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_branch_name.outputs.branch_name }} \
                                  --base_container ${{ steps.minimal-notebook-gpu.outputs.tag_pinned }} \
                                  --push \
                                  --prune_this

README.md

@@ -18,6 +18,7 @@ As summary of each step is given in their respective `readme.md` files. The note
 
 Github Actions CI is set up to build and deploy the images on merge to master.  Settings defining the base images used for CI (from `docker-stacks` and from `gpu-jupyter`) are defined in `build_settings.env`.  
 
+To reduce build times, the Github Actions CI workflow attempts to pull the latest version of each image from ACR before building.  In cases where cached layers can be reused, this can considerably reduce build time (at the cost of some data transfer time).  In general this appears to be of net benefit (if changing upstream images or `base-notebook` total build times will be ~5 minute slower, but if iterating on any other image build times will be ~10-60 minutes faster).
 
 ## Development Notes
 

base-notebook/readme.md

@@ -14,4 +14,4 @@ This provides a common notebook setup for both CPU and GPU paths, ensuring they'
 
 ## CI
 
-For CI, see `.github/workflows/build-cpu.yml` and `.github/workflows/build-gpu.yml`. They leverage the `.github/workflows/build_push.sh` wrapper for tagging and pushing all at once.
+For CI, see `.github/workflows/build-cpu.yml` and `.github/workflows/build-gpu.yml`.  They leverage the `.github/workflows/build_push.sh` wrapper for tagging, pushing, and caching all at once.

figures/dockerfile_hierarchy.xml

@@ -1,2 +1,2 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<mxfile host="app.diagrams.net" modified="2020-09-15T13:31:54.533Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36" etag="VA33pAITnzlLLc3zy3ud" version="13.6.9" type="google"><diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">7V1Zd5s4GP01Pqfz4BwWsz3GSZeZduZkJs20fVRAsakxclkSe379CJAwIGzkBeSFvAQECKF776dPnxYP1Lv58mMAFtM/kQO9gSI5y4F6P1AURTJH+F+SsspSZEWzspRJ4DokbZ3w6P4HSaJEUmPXgWHpxgghL3IX5UQb+T60o1IaCAL0Vr7tBXnlty7ABDIJjzbw2NRvrhNNs1RTMdbpn6A7mdI3yzr5vjmgN5MvCafAQW+FJPX9QL0LEIqyo/nyDnpJ7dF6+fb76pv3ZaZ//OPv8Bd4Gn/++te/wyyzD7s8kn9CAP1o76xfZ0ugP83GK/3zXw+zD/djdLscalnWr8CLSX2Rb41WtAKhg+uTnKIgmqIJ8oH3fp06DlDsOzB5jYTP1vd8QWiBE2Wc+BNG0YqQA8QRwknTaO6Rqy+u590hDwXpG1UoOxo0cHoYBWgGC1cs3VCBjq9kpUyKVkG5oYrIfSGKAxtuuU8lTAXBBEZb6m+U8wArCKI5jIIVfi6AHojc13LhAGHyJL9vjRY+IIDtwAuVAS9+jv0oHoykx0+3DJBlmN6mbgQfFyCthjcs/gokyI8IXjL+wvHEA2FIHs1goWpK7s6lIe2FJn58kbwJwwc8D3poEoB5jjIvqK8wiOCyUOUsLvSqTiRNjJpJTt/WBkKmqp8WjMNIOhzJWhoZvQw3y4tDhnpHMtxWyAJ47xxkz2AwDCNgz8LfBgmXdA9/wPg5wEeT9AiEcOijCD4jNLsYrTIa5CUDt1bzc2FitXqxbhYhh1hNkWLVm8Wqg3kiMP85XKS1VlXu3PXdOfB68e4u3lyUwsQrywwBevXmquRQryzVI96NfM3D5Rva7mK1Wby9XNdyVRXRclXYPo4IuTognKbP05MHEEUw8NMURZJrwcdIBKvvyUtvFI2e/yCFSE/ul6WzFTlrW+o0NNOo9ayyRWmdFvMQsTsgAljw0Ld7Z3sPAzCShBuAkQi9b5Su3CDd/QzF0o2+F45TG3GjkbP1m5KTrmyEqp2HjVBYGzFZxMOf8WKF653DQnx8eMLPr22DBH/FLs4uqcYkVJ3EvnczInKXRsQB0Hyx64yIbpvw+aVNYyGXjYXG6y2YrTn3POFs37lNhhXwmZ3UumuXoSlXv1KQp7SDPNcGpNTw32iH24+T8Q7kDcovwK/VoE/TuA0EecMDclNJEvapVqWpqjZB2YeSp4rjIpWM8gdXlYxpRllFMBlhFoFV4bZFckO4ucBmpbyKtLVYlds1vXQ7PsjevxZLjsAB+mGjI3doPneTYqUjCtJLgOb4nz1FIfQT01j0xzaayqrNZVOeQsz6tC5xEf1FnNvegjFnpIztWVQnXmoEfeTDisUkScBzJ4mmbCwZnLE6TqyjawPvllyYu46TuhJ1hrzsXrRlXoe57aSMMVj7qkg1ClNac8bY1nYcewlwlWi2dJ/yAtc8FNZQtoVL7hNTWIQHpBW22SOwsKHKS0ZGq/ZexCPDDuxRZIA9dX049CAIfNefXAlERtWmiYfIZF3ELjuYR/cPWWT37TK06GMqnD5m1g09oHdZ71xt8q52dx7VSkbSoc5jicxH8OlUtnE4fpe33bhZ11Ewhh3CR61U1jEveVxD7CZfDTy5H7wNnlGn8LDNfNXzGrJ9nmvCTK/aSfGYsUOJjE/GYnaVgjNM49TAs8Q6bTsO6B00KiDdGDTW+KNwScywAI0ANDtuh07J2+BvVWPOxr6OWyXqp8l8jtux3DL6+p7BXTOYd2Ara9SPzmBNaiAeL4M1OtpBM9I6ZrDcM1gMgw1eBh8607KeeLp+JAbrhmAGKyfF4FbnFlQYLAllsMo7VVhpJ/xzOTZY7RkshsG8NlhpyYswG4jHbYObpNA2g9nVM9cc2ZJvyriqNWO+3Xa1R2z8uI9tVWJbJ4gaGzDuo1ubolvm6cFn9O16G+1683puOt2YLxB2/PiWxdgSde8IF5NVla9tt+2Ch9avl8UaL4vb6V/p2tFYrMl6Navq9OG2WXxaYw1XxGKdl8Xt9LFM+Wgs1g3RLNb68QYxkQLqUYiLdTW6Ady2uNk5aZvF/ZiDIBZzLwdrK97VaEB5WWw0Oydts5iMO5SWKWV98aTfXOK3/itG9MIwTBmarMPA0C7XF2n//UO2OqOyKCPLGZc0y5wuwajoqGkxRXFZ6qaFrOexrKIS8azb/Mms6ey3tqpCq1vjfkQ6lJbQHI0M+VzhTbOLz5IMct2C527ZwAbEKWrhAvj7s+Hr1E1WV/0DF6hAgyxPbhocE1SGUlL61yncdUtWu4W734FxQ8XwjhFrQveT0thA+0Vuwtj+XhOG8KVAWr8N4zaJ8ezDKHRzKI1jd6h+J8ajCdYUviyGbkXdC7ZWhzyCVUQKlhaz341RiIAt4du76f1ujNt0ySPgrnYg31rMfj/GTgQrS8L3Y9MvYkPGQTGI37jovnWxc29zbgkVO8dG5/1+jC0bAOE/VWCdsQFoXcgWp5ANof1kne02vfOQDbykf8zbGz7P/WfyHY1PSFBs1KIXVEknPILaAHs3grIETxHbzZ05b6StQ6fJHLgnteB5VFcF9aFzSQ77Taaa4FTeSu4QdTrPhnJknlxDabDBoi276F0YHvro5PCwWIH005iEzVypHT7tdCqDxQYD+5krrcEtfNqaxc5cSfLGLbMdXpEdPoHfo2OnLSRWM3ZcdIHVX52822pMHJ+uf983m/67/plk9f3/</diagram></mxfile>
+<mxfile host="app.diagrams.net" modified="2020-11-25T18:47:58.638Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36" etag="C-ojRN1zDol2ZeOevRNX" version="13.10.3" type="device"><diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">7V3bdps4FP0ar9V5cBYXc3uMkzadaacrM2mm7aMCik2Nkcslsfv1Iy7CgLCRbUA4Ji+xhBBYe++jo6OLR/LNcn3ngdX8b2RBZyQJ1nok344kSRImOv4X5WySHHWiJBkzz7aSLHGb8WD/hmmmkOaGtgX9QsEAISewV8VME7kuNINCHvA89Fos9oyc4lNXYAapjAcTOHTuN9sK5kmuLmnb/I/Qns3Jk0XVSK4sASmcfhN/Diz0msuS34/kGw+hIPm0XN9AJ2o80i7f/tx8cz4v1Lu//vF/gcfpp69f/hsnlX045JbsK3jQDY6u+mWxBurjYrpRP325X3y4naLr9TjF8gU4Ydpe6XcNNqQBoYXbM00iL5ijGXKB836bO/VQ6FoweoyAU9synxFa4UwRZ/6EQbBJyQHCAOGsebB00qvPtuPcIAd58RNlKFoK1HC+H3hoAXNXDFWTgYqvMLZI2nI+Cj0T7iknp8QE3gwGe5prkpSLmiTHrrS97yBawsDb4AIedEBgvxQpCFImz7JyW7TwhxSwA3ghU+CFT6EbhKOJ8PDxmgKyCNPr3A7gwwrE7fKKtV+CBLlBipeIv+F05gDfT29NYCFqikpn0hCOQhPfvoqehPEEjgMdNPPA8mCUX6AXwPVeXMhVNZV0atP0NPm6NRAiUf08ZxwmwulIVvJKuzgZ7pNXrQxVXjLc99Y58N5ZyFxAb+wHwFz4f4wiLqkO/kbTJw9/msWfgA/HLgrgE0KLQavMWs3S3MRqDGLNibBWrHqvxKrWi1UFy0hg7pO/ilutrNyl7dpL4AziPVy8mSi5iVcUKQJcpHp1RvWSYVxP5KufLl/ftFeb3eId5LqVqyzxlqtEj3F4yNUC/jy+nyTuQRBAz41zJEGsBB+D422+Rw+9khSS/pG+RJy4XRdSmzTVsNSJhGu1Ttq2J1on732K2C0QACx46JqDs32EAZgI3A3AhIfed0pXrJHucYZibQffc59jG3GlpKntk6JESzZCVs7URki0jZitwvHPcLXB7c5gIe7uH/H9W9sgwF+hjauL2jUKVUeh78OMiNilEbEA1J/NKiOimjp8em7TWIhFY6Gwegt6a849Szjbta6jaQWcMqNWt80iNMXml3LyFA6Q59aAFDr+K+V0+8HLOxBZlZ+DX6lAn+QxG4j0CffIjiWZsk82Sl1VuQtKvnl6V35epFRRduOmVDGpKGkZqiLMIrDJFVtFBfzdL6yX3lcS9r5WqbiiForjD8nzt2LJEDhBP3R05AYtl3b0WvGMgvDsoSX+Z86RD93INOb9sZ2msmxz6ZxHH7M+bkv8iu4qzGxvzphTUsZ2KqgSLzGCLnJhyWKmWcCxZ5GmTKwhXLE8jayebQLnOr2wtC0rdiWqDHnRvWjLvI4z20kYo9H2VRIqFCa15ozRve00tJ0IuVI4W7iNiYGbHnLrKdsCJnOKCS7cI9IS3e8RXOhg5VuGRimPX/hDQ0/tZdAAc267cOxA4Lm2O7sQjLSyWeOPkU57iV2OMRt3EWlk2x411LuZEqObSUaizQ0wq/2rXQ7W4f6jXKpIONV/LJC5AbdOpruH5ke97YbOug6EUezgPnEl0755wecaY0/5YuDJXOF98Ew6hYfu58uu15ge9lwSZmrZTvLHjJ5NpHwyGrOLFJyma30Dz+DrtB04p3fSxIBwpZFw44/cpU5mBsiYv95xa3xV3g5/qxx21o513EqBP0Vkc9yacsvI4wcGt8xg1rkt0oe3zWBFqCEeK4MVMuFBKlI6ZrA4MLgTBmusDG58sWU18VS1IQarGmcGS71icKvLC0oMFrpksMy6WljqKPzzdmywPDC4Ewaz2mCpKy9CryEesw2uk0LbDKY30BQjW+YFDbQnmnhVxFWumPbtdqg9oePHDLGtS0JNlXuIGh0wPiS6dUnwabreP/i0oV/vVXyLdFOtx7cMypbIR0e4qKrKfG27b+c8tX4xLGaNcU06Gl+pSmMsVkS1XFV5BXHbLO7XXMPbZTHrGGvS0VyDLjbGYlXjzWJlmG/ohMXEUehRrKvWDWC2xfXOSdssHuYcumEx846wzuJdtQaUlcVavXPSNovTUUdhp1IyFo/GzQV+q79CRC6M/Zih0VYMjPV6e5GM3z8kGzRK+zKSmvGbJpWTXRglHdXtp8jvTN21l/U8dlaUIp5V5z/pFYP91jZWKFXb3BukQ2EXTWNkyNYK71pdfJZkEKv2PHfLBjogTlDzV8A9ng1f53a0wepfuEI5GiR1MtOgSVApSgnxX6dwV+1a7RbuyzuEsbodWOeIlX4dKaXQgfbhHMajjpvQuG8FUoaTGAsSqz+KsV/nQykMB0QNhzE2d3Aq920xynAaY0GH9YKVeiVY8t7DgYxcBGxwP+FNHQ5kLOiyXsDcDiHf+97DkYydCFYUuB/Jpr6JMxlH+SB+7ab7psXOfNK50S+xM5x1PhzJ2LIB4P5rBcYZG4CmhWwwClnr1zhZpYdN7xxkAicaH7OOhs/z/JnsUOMeCYqOWlyqoDTWcawm9kpQBuclYoe5M2eFtNH4MpkTj6XmvI7qLUPd+FqS036WqSI4lfWSB0SdzrOjnOi96yiJ0d+3p+XN4qFOeoeHQQtkWMbEbeVK5fRpp0sZDDoYOKxcaQ1u7svWDHrlSlQ37plN/4LscA9+ko5ethBZzdCy0Rts/vLi3VZj4ji5/YnfZPnv9oeS5ff/Aw==</diagram></mxfile>

geomatics-notebook/readme.md

@@ -4,4 +4,4 @@ This is a Dockerfile which adds geomatics-related R packages.  Typical usage for
 
 ## CI
 
-For CI, see `.github/workflows/build-cpu.yml`.  It leverages the `.github/workflows/build_push.sh` wrapper for tagging and pushing all at once. 
+For CI, see `.github/workflows/build-cpu.yml`.  It leverages the `.github/workflows/build_push.sh` wrapper for tagging, pushing and caching all at once.

machine-learning-notebook/readme.md

@@ -4,4 +4,4 @@ This is a Dockerfile which adds Tensorflow and Pytorch. Typical usage for this D
 
 ## CI
 
-For CI, see `.github/workflows/build-cpu.yml` and `.github/workflows/build-gpu.yml`. They leverage the `.github/workflows/build_push.sh` wrapper for tagging and pushing all at once.
+For CI, see `.github/workflows/build-cpu.yml` and `.github/workflows/build-gpu.yml`. They leverage the `.github/workflows/build_push.sh` wrapper for tagging, pushing and caching all at once.

minimal-notebook/readme.md

@@ -10,4 +10,4 @@ Typical usage for this Dockerfile is to be built off a `base-notebook-cpu` or `b
 
 ## CI
 
-For CI, see `.github/workflows/build-cpu.yml` and `.github/workflows/build-gpu.yml`.  They leverage the `.github/workflows/build_push.sh` wrapper for tagging and pushing all at once. 
+For CI, see `.github/workflows/build-cpu.yml` and `.github/workflows/build-gpu.yml`.  They leverage the `.github/workflows/build_push.sh` wrapper for tagging, pushing and caching all at once. 

r-studio/readme.md

@@ -5,4 +5,4 @@ This is a Dockerfile which adds R-studio and some additional R packages.  Typica
 
 ## CI
 
-For CI, see `.github/workflows/build-cpu.yml`.  It leverages the `.github/workflows/build_push.sh` wrapper for tagging and pushing all at once. 
+For CI, see `.github/workflows/build-cpu.yml`.  It leverages the `.github/workflows/build_push.sh` wrapper for tagging, pushing and caching all at once. 

scripts/build_push.sh

@@ -5,15 +5,17 @@ set -e
 USAGE_MESSAGE="
 This script handles building, tagging, and (optionally) pushing an image to a docker repo. It builds the image requested, adds two tags (one intended for version pinning such as a git sha, and one intended for something like 'latest') 
 and emits the github action 'tag_pinned' when when run as a step in a github action.  It optionally can: 
+* pull an existing image to use as a layer cache
 * pass a build-arg 'BASE_CONTAINER' to the 'docker build' command, indicating the container to build from
 * prune either all docker images/containers or the images built since starting this script (typically just the images built here, but not guaranteed if other processes run in parallel)
-Usage: $0 --registry REGISTRY --image_name IMAGE_NAME --tag_pinned GITHUB_SHA --tag_latest TAG_LATEST [--base_container BASE_CONTAINER] [--push] [--prune]
+Usage: $0 --registry REGISTRY --image_name IMAGE_NAME --tag_pinned GITHUB_SHA --tag_latest TAG_LATEST [--base_container BASE_CONTAINER] [--cache-from FULL_IMAGE_PATH] [--push] [--prune]
 Where:
     registry: container registry full path, such as myRegistry.azurecr.io
     image_name: name of the image to be pushed
     tag_pinned: tag used for the 'pinned' version (typically a github sha)
-    tag_latest: tag used for the latest version (typically branch name or master)
+    tag_latest: tag used for the latest version, both to pull a recent cache and push back to registry
     base_container: (optional) if set, pass this value to docker as a build arg (eg: '--build-arg BASE_CONTAINER=THIS_VALUE')
+    cache_from: (optional) if set, will pull this image to help caching.  Typically set to a recently built version of this image, such as myRegistry.azurecr.io/IMAGE_NAME:TAG_LATEST
     push: (optional) if set, will push all products to registry.  Default is unset
     prune_all: (optional) if set, will 'docker system prune -f -a' after build.  Default is unset
     prune_this: (optional) if set, will remove all images built since this job started (like prune_all but only for products of this job)
@@ -23,6 +25,7 @@ Where:
 PUSH=""
 PRUNE_ALL=""
 PRUNE_THIS=""
+CACHE_FROM=""
 BASE_CONTAINER=""
 REGISTRY=""
 REMOVE_DANGLING="true"
@@ -40,6 +43,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   --tag_pinned) GITHUB_SHA="$2"; shift;;
   --tag_latest) LATEST="$2"; shift;;
   --base_container) BASE_CONTAINER="$2"; shift;;
+  --cache_from) CACHE_FROM="$2"; shift ;;
   --push) PUSH="true"; ;;
   --prune_all) PRUNE_ALL="true"; ;;
   --prune_this) PRUNE_THIS="true"; ;;
@@ -74,6 +78,15 @@ fi
 
 BUILD_COMMAND="docker build -t $TAG_PINNED"
 
+# Initialize layer caching by pulling the cache_from image
+if [ -z "$CACHE_FROM" ]; then
+    echo "Skipping cache pull"
+else
+    echo "Pulling cache image"
+    docker pull "$CACHE_FROM" || true
+    BUILD_COMMAND="$BUILD_COMMAND --cache-from $CACHE_FROM"
+fi
+
 echo "Building image"
 if [ -z "$BASE_CONTAINER" ]; then
     BUILD_COMMAND="$BUILD_COMMAND ."