rocker-org · cboettig · Jul 27, 2021 · Jun 26, 2021 · Jun 26, 2021 · Jun 29, 2021
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
@@ -5,7 +5,7 @@ on:
     branches:
       - "master"
     paths:
-      - ".github/workflows/buildmatrix/**"
+      - "build/matrix/**"
       - "scripts/**"
       - "dockerfiles/Dockerfile_r-ver_*"
       - "dockerfiles/Dockerfile_rstudio_*"
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v2
       - id: set-json
         run: |
-          JSON=.github/workflows/buildmatrix/latest.json
+          JSON=build/matrix/latest.json
           echo ::set-output name=json::${JSON}
           echo ${JSON}
       - id: set-matrix
@@ -49,12 +49,14 @@ jobs:
           docker-compose -f compose/shiny-${{ matrix.r_version }}.yml build
           docker-compose -f compose/geospatial-${{ matrix.r_version }}.yml build
           docker-compose -f compose/binder-${{ matrix.r_version }}.yml build
+          docker-compose -f compose/ml-cuda10.1-${{ matrix.r_version }}.yml build
       - name: Push the tagged Docker image
         run: |
           docker-compose -f compose/core-${{ matrix.r_version }}.yml push
           docker-compose -f compose/shiny-${{ matrix.r_version }}.yml push
           docker-compose -f compose/geospatial-${{ matrix.r_version }}.yml push
           docker-compose -f compose/binder-${{ matrix.r_version }}.yml push
+          docker-compose -f compose/ml-cuda10.1-${{ matrix.r_version }}.yml push
       - name: Push latest tagged Docker image
         if: matrix.r_latest == true
         run: |
@@ -66,6 +68,8 @@ jobs:
           docker image tag rocker/shiny-verse:${{ matrix.r_version }} rocker/shiny-verse:latest
           docker image tag rocker/geospatial:${{ matrix.r_version }} rocker/geospatial:latest
           docker image tag rocker/binder:${{ matrix.r_version }} rocker/binder:latest
+          docker image tag rocker/ml:${{ matrix.r_version }} rocker/ml:latest
+          docker image tag rocker/ml-verse:${{ matrix.r_version }} rocker/ml-verse:latest
           docker image push rocker/r-ver:latest
           docker image push rocker/rstudio:latest
           docker image push rocker/tidyverse:latest
@@ -74,3 +78,5 @@ jobs:
           docker image push rocker/shiny-verse:latest
           docker image push rocker/geospatial:latest
           docker image push rocker/binder:latest
+          docker image push rocker/ml:latest
+          docker image push rocker/ml-verse:latest
diff --git a/.github/workflows/dockerfiles.yml b/.github/workflows/dockerfiles.yml
@@ -16,7 +16,9 @@ jobs:
           install2.r --error --skipinstalled -r https://r-lib.github.io/p/pak/dev/ -n -1 pak
       - name: Make changes to pull request
         run: |
-          ./make-stacks.R
+          ./build/sync-template-vars.R
+          ./build/make-stacks.R
+          make clean
           make setup
       - name: Create Pull Request
         id: cpr

diff --git a/.github/workflows/ml-10.1.yml b/.github/workflows/ml-10.1.yml
diff --git a/.mdlrc b/.mdlrc
@@ -1 +1 @@
-rules "~MD036", "~MD013", "~MD024", "~MD045", "~MD041", "~MD002"
+rules "~MD036", "~MD013", "~MD024", "~MD045", "~MD041", "~MD002", "~MD029"
diff --git a/Makefile b/Makefile
@@ -25,9 +25,9 @@ latest: clean setup core-$(LATEST_TAG) geospatial-$(LATEST_TAG) binder-$(LATEST_
 
 
 setup: $(COMPOSEFILES)
-$(COMPOSEFILES): make-dockerfiles.R write-compose.R $(STACKFILES)
-	./make-dockerfiles.R
-	./write-compose.R
+$(COMPOSEFILES): ./build/make-dockerfiles.R ./build/write-compose.R $(STACKFILES)
+	./build/make-dockerfiles.R
+	./build/write-compose.R
 
 
 ## Builds all stacks

diff --git a/README.md b/README.md
@@ -38,9 +38,9 @@ image            | description                               | size   | pull
 
 ### Modifying and extending images in the new architecture
 
-In our new build system, pre-built images are defined with JSON files under the `stacks/` folder
-in this repository.  Each file defines a series of related images.  The `.R` files in the base
-folder use these to create the actual Dockerfiles under `dockerfiles` and images are built using the `Makefile`.
+In our new build system, pre-built images are defined with JSON files under [the `stacks/` folder](./stacks)
+in this repository.  Each file defines a series of related images.  The `.R` files in [the `build/`
+folder](./build) use these to create the actual Dockerfiles under [the `dockerfiles/` folder](./dockerfiles) and images are built using the `Makefile`.
 These Dockerfiles are server as examples of how to build your own custom images.
 
 Importantly, we have moved as much of the detailed install logic out of Dockerfiles and into standalone scripts, or "modules", under the `scripts` directory.  These files are available in all Docker images, under a top-level `/rocker_scripts` directory.  This allows users to extend images by selecting additional modules to install on top of any pre-built images.  For instance, if one wishes to install Shiny Server and Binder dependencies on top of a base of `rstudio:3.6.3`, one could write a simple Dockerfile as follows:

diff --git a/build/README.md b/build/README.md
@@ -0,0 +1,14 @@
+# Build scripts for container definition files
+
+This folder contains scripts and other files to create (or assist in creating) definition files needed to build Rocker container images.
+
+## Overview
+
+The Dockerfiles for pre-build images will be generated by executing the scripts in the following order. It will also generate the `docker-compose.yml` files that can be used to build the container images.
+
+1. `./build/sync-template-vars.R`
+2. `./build/make-stacks.R`
+3. `./build/make-dockerfiles.R`
+4. `./build/write-compose.R`
+
+These scripts are run daily by GitHub Actions and automatically create a Pull Request if the run results in a variance from the default branch.
diff --git a/make-dockerfiles.R → build/make-dockerfiles.R b/make-dockerfiles.R → build/make-dockerfiles.R
diff --git a/make-stacks.R → build/make-stacks.R b/make-stacks.R → build/make-stacks.R
@@ -199,6 +199,32 @@ write_stack_binder <- function(r_version, ubuntu_version, cran, rstudio_version,
     message(output_path)
 }
 
+write_stack_mlcuda10 <- function(r_version, ubuntu_version, cran, rstudio_version, ctan_repo) {
+    template <- jsonlite::read_json("stacks/ml-cuda10.1-devel.json")
+
+    output_path <- paste0("stacks/ml-cuda10.1-", r_version, ".json")
+
+    template$TAG <- paste0(r_version, "-cuda10.1")
+    # rocker/r-ver:X.Y.Z-cuda10.1
+    template$stack[[1]]$FROM <- paste0("rocker/r-ver:", r_version)
+    # rocker/ml:X.Y.Z-cuda10.1
+    template$stack[[2]]$FROM <- paste0("rocker/r-ver:", r_version, "-cuda10.1")
+    template$stack[[2]]$ENV$RSTUDIO_VERSION <- rstudio_version
+    # rocker/ml-verse:X.Y.Z-cuda10.1
+    template$stack[[3]]$FROM <- paste0("rocker/ml:", r_version, "-cuda10.1")
+    template$stack[[3]]$ENV$CTAN_REPO <- ctan_repo
+    # rocker/ml:X.Y.Z
+    template$stack[[4]]$FROM <- paste0("rocker/ml:", r_version, "-cuda10.1")
+    template$stack[[4]]$TAG <- paste0(r_version)
+    # rocker/ml-verse:X.Y.Z
+    template$stack[[5]]$FROM <- paste0("rocker/ml-verse:", r_version, "-cuda10.1")
+    template$stack[[5]]$TAG <- paste0(r_version)
+
+    jsonlite::write_json(template, output_path, pretty = TRUE, auto_unbox = TRUE)
+
+    message(output_path)
+}
+
 
 df_args <- .r_versions_data(min_version = 4.0) %>%
     dplyr::rowwise() %>%
@@ -228,7 +254,7 @@ df_args %>%
     {
         list(include = .)
     } %>%
-    jsonlite::write_json(".github/workflows/buildmatrix/latest.json", pretty = TRUE, auto_unbox = TRUE)
+    jsonlite::write_json("build/matrix/latest.json", pretty = TRUE, auto_unbox = TRUE)
 
 message("\nstart writing stack files.")
 
@@ -240,6 +266,7 @@ devnull <- df_args %>%
         write_stack_shiny(df[1], df[2], df[3], df[4], df[5])
         write_stack_geospatial(df[1], df[2], df[3], df[4], df[5])
         write_stack_binder(df[1], df[2], df[3], df[4], df[5])
+        write_stack_mlcuda10(df[1], df[2], df[3], df[4], df[5])
     })
 
 message("make-stacks.R done!\n")
diff --git a/.github/workflows/buildmatrix/latest.json → build/matrix/latest.json b/.github/workflows/buildmatrix/latest.json → build/matrix/latest.json
diff --git a/build/sync-template-vars.R b/build/sync-template-vars.R
@@ -0,0 +1,20 @@
+#!/usr/bin/env Rscript
+
+library(jsonlite)
+
+template_core <- jsonlite::read_json("stacks/core-devel.json")
+template_shiny <- jsonlite::read_json("stacks/shiny-devel.json")
+template_mlcuda_10 <- jsonlite::read_json("stacks/ml-cuda10.1-devel.json")
+
+# Copy S6_VERSION from core to others.
+s6_ver <- template_core$stack[[2]]$ENV$S6_VERSION
+
+template_mlcuda_10$stack[[2]]$ENV$S6_VERSION <- s6_ver
+template_shiny$stack[[1]]$ENV$S6_VERSION <- s6_ver
+
+
+# Rewrite template files.
+jsonlite::write_json(template_shiny, "stacks/shiny-devel.json", pretty = TRUE, auto_unbox = TRUE)
+jsonlite::write_json(template_mlcuda_10, "stacks/ml-cuda10.1-devel.json", pretty = TRUE, auto_unbox = TRUE)
+
+message("sync-template-vers.R done!\n")
diff --git a/write-compose.R → build/write-compose.R b/write-compose.R → build/write-compose.R
diff --git a/compose/ml-cuda10.1-4.1.0.yml b/compose/ml-cuda10.1-4.1.0.yml
@@ -19,3 +19,17 @@ services:
     build:
       context: ..
       dockerfile: dockerfiles/Dockerfile_ml-verse_4.1.0-cuda10.1
+  ml-4.1.0:
+    image: rocker/ml:4.1.0
+    depends_on:
+    - ml-verse-4.1.0-cuda10.1
+    build:
+      context: ..
+      dockerfile: dockerfiles/Dockerfile_ml_4.1.0
+  ml-verse-4.1.0:
+    image: rocker/ml-verse:4.1.0
+    depends_on:
+    - ml-4.1.0
+    build:
+      context: ..
+      dockerfile: dockerfiles/Dockerfile_ml-verse_4.1.0
diff --git a/dockerfiles/Dockerfile_ml-verse_4.0.5-cuda10.1 b/dockerfiles/Dockerfile_ml-verse_4.0.5-cuda10.1
@@ -5,6 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
+ENV CTAN_REPO=http://www.texlive.info/tlnet-archive/2021/05/17/tlnet
 
 
 RUN /rocker_scripts/install_verse.sh

diff --git a/dockerfiles/Dockerfile_ml-verse_4.1.0 b/dockerfiles/Dockerfile_ml-verse_4.1.0
@@ -0,0 +1,14 @@
+FROM rocker/ml-verse:4.1.0-cuda10.1
+
+LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
+      org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \
+      org.opencontainers.image.vendor="Rocker Project" \
+      org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
+
+
+
+
+
+
+
+
diff --git a/dockerfiles/Dockerfile_ml-verse_4.1.0-cuda10.1 b/dockerfiles/Dockerfile_ml-verse_4.1.0-cuda10.1
@@ -1,10 +1,11 @@
 FROM rocker/ml:4.1.0-cuda10.1
 
-LABEL org.label-schema.license="GPL-2.0" \
-      org.label-schema.vcs-url="https://github.com/rocker-org/rocker-versioned" \
-      org.label-schema.vendor="Rocker Project" \
-      maintainer="Carl Boettiger <cboettig@ropensci.org>"
+LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
+      org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \
+      org.opencontainers.image.vendor="Rocker Project" \
+      org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
+ENV CTAN_REPO=http://mirror.ctan.org/systems/texlive/tlnet
 
 
 RUN /rocker_scripts/install_verse.sh

diff --git a/dockerfiles/Dockerfile_ml-verse_devel-cuda10.1 b/dockerfiles/Dockerfile_ml-verse_devel-cuda10.1
@@ -5,6 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
+ENV CTAN_REPO=http://mirror.ctan.org/systems/texlive/tlnet
 
 
 RUN /rocker_scripts/install_verse.sh

diff --git a/dockerfiles/Dockerfile_ml_4.0.5-cuda10.1 b/dockerfiles/Dockerfile_ml_4.0.5-cuda10.1
@@ -5,8 +5,8 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
-ENV S6_VERSION=v1.21.7.0
-ENV RSTUDIO_VERSION=latest
+ENV S6_VERSION=v2.1.0.2
+ENV RSTUDIO_VERSION=1.4.1106
 ENV PANDOC_VERSION=default
 ENV TENSORFLOW_VERSION=gpu
 ENV KERAS_VERSION=default

diff --git a/dockerfiles/Dockerfile_ml_4.1.0 b/dockerfiles/Dockerfile_ml_4.1.0
@@ -0,0 +1,14 @@
+FROM rocker/ml:4.1.0-cuda10.1
+
+LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
+      org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \
+      org.opencontainers.image.vendor="Rocker Project" \
+      org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
+
+
+
+
+
+
+
+
diff --git a/dockerfiles/Dockerfile_ml_4.1.0-cuda10.1 b/dockerfiles/Dockerfile_ml_4.1.0-cuda10.1
@@ -1,12 +1,12 @@
 FROM rocker/r-ver:4.1.0-cuda10.1
 
-LABEL org.label-schema.license="GPL-2.0" \
-      org.label-schema.vcs-url="https://github.com/rocker-org/rocker-versioned" \
-      org.label-schema.vendor="Rocker Project" \
-      maintainer="Carl Boettiger <cboettig@ropensci.org>"
+LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
+      org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \
+      org.opencontainers.image.vendor="Rocker Project" \
+      org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
-ENV S6_VERSION=v1.21.7.0
-ENV RSTUDIO_VERSION=latest
+ENV S6_VERSION=v2.1.0.2
+ENV RSTUDIO_VERSION=1.4.1717
 ENV PANDOC_VERSION=default
 ENV TENSORFLOW_VERSION=gpu
 ENV KERAS_VERSION=default

diff --git a/dockerfiles/Dockerfile_ml_devel-cuda10.1 b/dockerfiles/Dockerfile_ml_devel-cuda10.1
@@ -5,7 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
-ENV S6_VERSION=v1.21.7.0
+ENV S6_VERSION=v2.1.0.2
 ENV RSTUDIO_VERSION=latest
 ENV PANDOC_VERSION=default
 ENV TENSORFLOW_VERSION=gpu

diff --git a/dockerfiles/Dockerfile_r-ver_4.0.5-cuda10.1 b/dockerfiles/Dockerfile_r-ver_4.0.5-cuda10.1
@@ -6,6 +6,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
 ENV CUDA_VERSION=10.1.243
+ENV CUDA_PKG_VERSION=10-1=$CUDA_VERSION-1
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 ENV NVIDIA_REQUIRE_CUDA=cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411

diff --git a/dockerfiles/Dockerfile_r-ver_4.1.0-cuda10.1 b/dockerfiles/Dockerfile_r-ver_4.1.0-cuda10.1
@@ -1,23 +1,23 @@
 FROM rocker/r-ver:4.1.0
 
-LABEL org.label-schema.license="GPL-2.0" \
-      org.label-schema.vcs-url="https://github.com/rocker-org/rocker-versioned" \
-      org.label-schema.vendor="Rocker Project" \
-      maintainer="Carl Boettiger <cboettig@ropensci.org>"
+LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
+      org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \
+      org.opencontainers.image.vendor="Rocker Project" \
+      org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
 ENV CUDA_VERSION=10.1.243
+ENV CUDA_PKG_VERSION=10-1=$CUDA_VERSION-1
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 ENV NVIDIA_REQUIRE_CUDA=cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411
 ENV CUDA_HOME=/usr/local/cuda
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_HOME/lib64:$CUDA_HOME/extras/CUPTI/lib64:$CUDA_HOME/lib64/libnvblas.so:
-ENV LIBRARY_PATH=/usr/local/cuda/lib64/stubs
 ENV NVBLAS_CONFIG_FILE=/etc/nvblas.conf
 ENV WORKON_HOME=/opt/venv
 ENV PYTHON_VENV_PATH=/opt/venv/reticulate
 ENV PYTHON_CONFIGURE_OPTS=--enable-shared
-ENV RETICULATE_MINICONDA_ENABLED=FALSE
-ENV PATH=${PYTHON_VENV_PATH}/bin:${CUDA_HOME}/bin:/usr/local/nviida/bin:${PATH}:/usr/local/texlive/bin/x86_64-linux
+ENV RETICULATE_AUTOCONFIGURE=0
+ENV PATH=${PYTHON_VENV_PATH}/bin:$PATH:${CUDA_HOME}/bin:/usr/local/texlive/bin/x86_64-linux
 
 
 RUN /rocker_scripts/install_cuda-10.1.sh

diff --git a/dockerfiles/Dockerfile_r-ver_devel-cuda10.1 b/dockerfiles/Dockerfile_r-ver_devel-cuda10.1
@@ -17,7 +17,7 @@ ENV WORKON_HOME=/opt/venv
 ENV PYTHON_VENV_PATH=/opt/venv/reticulate
 ENV PYTHON_CONFIGURE_OPTS=--enable-shared
 ENV RETICULATE_AUTOCONFIGURE=0
-ENV PATH=${PYTHON_VENV_PATH}/bin:${PATH}:${CUDA_HOME}/bin
+ENV PATH=${PYTHON_VENV_PATH}/bin:$PATH:${CUDA_HOME}/bin:/usr/local/texlive/bin/x86_64-linux
 
 
 RUN /rocker_scripts/install_cuda-10.1.sh

diff --git a/dockerfiles/Dockerfile_shiny_4.0.5 b/dockerfiles/Dockerfile_shiny_4.0.5
@@ -5,7 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
-ENV S6_VERSION=v1.21.7.0
+ENV S6_VERSION=v2.1.0.2
 ENV SHINY_SERVER_VERSION=latest
 ENV PANDOC_VERSION=default
 

diff --git a/dockerfiles/Dockerfile_shiny_4.1.0 b/dockerfiles/Dockerfile_shiny_4.1.0
@@ -5,7 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
-ENV S6_VERSION=v1.21.7.0
+ENV S6_VERSION=v2.1.0.2
 ENV SHINY_SERVER_VERSION=latest
 ENV PANDOC_VERSION=default
 

diff --git a/dockerfiles/Dockerfile_shiny_devel b/dockerfiles/Dockerfile_shiny_devel
@@ -5,7 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <cboettig@ropensci.org>"
 
-ENV S6_VERSION=v1.21.7.0
+ENV S6_VERSION=v2.1.0.2
 ENV SHINY_SERVER_VERSION=latest
 ENV PANDOC_VERSION=default
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		rules "~MD036", "~MD013", "~MD024", "~MD045", "~MD041", "~MD002"
		rules "~MD036", "~MD013", "~MD024", "~MD045", "~MD041", "~MD002", "~MD029"