Feature: Add Batch Service Built-In Container support (#291)

* Replaced Virtual Configuration

* Added containerSettings in addTask helper function

* Using new Dockerfile with blobxfer and R

* Removed container settings

* Fixed containerConfiguration lintr

* Fixed description dependency

* Fixed cluster setting

* Fixed R packages installation on pool level

Author: brnleehng

DESCRIPTION

@@ -28,5 +28,5 @@ Suggests:
     plyr,
     lintr
 Remotes:
-  Azure/rAzureBatch@v0.6.1
+  Azure/rAzureBatch@v0.6.2
 RoxygenNote: 6.0.1

R/batch-api.R

@@ -17,7 +17,6 @@ BatchUtilities <- R6::R6Class(
       userOutputFiles <- args$outputFiles
       containerImage <- args$containerImage
 
-      resultFile <- paste0(taskId, "-result", ".rds")
       accountName <- storageClient$authentication$name
 
       resourceFiles <- NULL
@@ -46,8 +45,15 @@ BatchUtilities <- R6::R6Class(
       # Otherwise just leave it empty
       commands <- c()
 
+      containerSettings <- list(
+        imageName = containerImage,
+        containerRunOptions = "--rm"
+      )
+
       if (!is.null(cloudCombine)) {
         assign("cloudCombine", cloudCombine, .doAzureBatchGlobals)
+        containerSettings$imageName <- "brianlovedocker/doazureparallel-merge-dockerfile:0.12.1"
+
         copyCommand <- sprintf(
           "%s %s %s --download --saskey $BLOBXFER_SASKEY --remoteresource . --include results/*.rds --endpoint %s",
           accountName,
@@ -56,9 +62,7 @@ BatchUtilities <- R6::R6Class(
           config$endpointSuffix
         )
 
-        downloadCommand <-
-          dockerRunCommand("alfpark/blobxfer:0.12.1", copyCommand, "blobxfer", FALSE)
-        commands <- c(downloadCommand)
+        commands <- c(paste("blobxfer", copyCommand))
       }
 
       exitConditions <- NULL
@@ -108,7 +112,7 @@ BatchUtilities <- R6::R6Class(
 
       commands <-
         c(commands,
-          dockerRunCommand(containerImage, rCommand))
+          rCommand)
 
       commands <- linuxWrapCommands(commands)
 
@@ -141,7 +145,8 @@ BatchUtilities <- R6::R6Class(
         commandLine = commands,
         dependsOn = dependsOn,
         outputFiles = outputFiles,
-        exitConditions = exitConditions
+        exitConditions = exitConditions,
+        containerSettings = containerSettings
       )
     },
     addJob = function(jobId,
@@ -250,12 +255,13 @@ BatchUtilities <- R6::R6Class(
 
         virtualMachineConfiguration <- list(
           imageReference = list(
-            publisher = "Canonical",
-            offer = "UbuntuServer",
-            sku = "16.04-LTS",
+            publisher = "microsoft-azure-batch",
+            offer = "ubuntu-server-container",
+            sku = "16-04-lts",
             version = "latest"
           ),
-          nodeAgentSKUId = "batch.node.ubuntu 16.04"
+          nodeAgentSKUId = "batch.node.ubuntu 16.04",
+          containerConfiguration = args$containerConfiguration
         )
 
         response <- batchClient$poolOperations$addPool(

R/cluster.R

@@ -92,61 +92,54 @@ makeCluster <-
     installGithubCommand <- NULL
     installBioconductorCommand <- NULL
 
+    packages <- c()
     if (!is.null(poolConfig$rPackages) &&
         !is.null(poolConfig$rPackages$cran) &&
         length(poolConfig$rPackages$cran) > 0) {
       installCranCommand <-
         getPoolPackageInstallationCommand("cran", poolConfig$rPackages$cran)
+      packages <- c(packages, installCranCommand)
     }
 
     if (!is.null(poolConfig$rPackages) &&
         !is.null(poolConfig$rPackages$github) &&
         length(poolConfig$rPackages$github) > 0) {
       installGithubCommand <-
         getPoolPackageInstallationCommand("github", poolConfig$rPackages$github)
+      packages <- c(packages, installGithubCommand)
     }
 
     if (!is.null(poolConfig$rPackages) &&
         !is.null(poolConfig$rPackages$bioconductor) &&
         length(poolConfig$rPackages$bioconductor) > 0) {
       installBioconductorCommand <-
         getPoolPackageInstallationCommand("bioconductor", poolConfig$rPackages$bioconductor)
-    }
-
-    packages <- c()
-    if (!is.null(installCranCommand)) {
-      packages <- c(packages, installCranCommand)
-    }
-    if (!is.null(installGithubCommand)) {
-      packages <- c(packages, installGithubCommand)
-    }
-    if (!is.null(installBioconductorCommand)) {
       packages <- c(packages, installBioconductorCommand)
     }
 
     if (length(packages) == 0) {
       packages <- NULL
     }
 
-    commandLine <- NULL
-
     # install docker
+    containerConfiguration <- list(
+      type = "docker"
+    )
+
     dockerImage <- "rocker/tidyverse:latest"
     if (!is.null(poolConfig$containerImage) &&
         nchar(poolConfig$containerImage) > 0) {
       dockerImage <- poolConfig$containerImage
     }
 
+    containerConfiguration$containerImageNames <-
+      list(dockerImage,
+           "brianlovedocker/doazureparallel-merge-dockerfile:0.12.1")
+
     config$containerImage <- dockerImage
-    installAndStartContainerCommand <- "cluster_setup.sh"
 
     # Note: Revert it to master once PR is approved
     dockerInstallCommand <- c(
-      paste0(
-        "wget https://raw.githubusercontent.com/Azure/doAzureParallel/",
-        "master/inst/startup/cluster_setup.sh"
-      ),
-      "chmod u+x cluster_setup.sh",
       paste0(
         "wget https://raw.githubusercontent.com/Azure/doAzureParallel/",
         "master/inst/startup/install_bioconductor.R"
@@ -155,11 +148,12 @@ makeCluster <-
         "wget https://raw.githubusercontent.com/Azure/doAzureParallel/",
         "master/inst/startup/install_custom.R"
       ),
-      "chmod u+x install_bioconductor.R",
-      installAndStartContainerCommand
+      "chmod u+x install_bioconductor.R"
     )
 
-    commandLine <- dockerInstallCommand
+    commandLine <- c(dockerInstallCommand,
+                    'mkdir -p $AZ_BATCH_NODE_SHARED_DIR/R/packages',
+                    'chmod -R 0777 $AZ_BATCH_NODE_SHARED_DIR/R')
 
     # log into private registry if registry credentials were provided
     if (!is.null(config$dockerAuthentication) &&
@@ -171,14 +165,13 @@ makeCluster <-
       password <- config$dockerAuthentication$password
       registry <- config$dockerAuthentication$registry
 
-      loginCommand <- dockerLoginCommand(username, password, registry)
-      commandLine <- c(commandLine, loginCommand)
+      containerConfiguration$containerRegistries <- list(
+        list(password = password,
+             username = username,
+             registryServer = registry)
+      )
     }
 
-    # pull docker image
-    pullImageCommand <- dockerPullCommand(dockerImage)
-    commandLine <- c(commandLine, pullImageCommand)
-
     if (!is.null(poolConfig$commandLine)) {
       commandLine <- c(commandLine, poolConfig$commandLine)
     }
@@ -235,7 +228,8 @@ makeCluster <-
       environmentSettings = environmentSettings,
       resourceFiles = resourceFiles,
       commandLine = commandLine,
-      networkConfiguration = networkConfiguration
+      networkConfiguration = networkConfiguration,
+      containerConfiguration = containerConfiguration
     )
 
     if (nchar(response) > 0) {

inst/startup/merger.R

@@ -36,10 +36,6 @@ azbatchenv <-
 
 setwd(batchTaskWorkingDirectory)
 
-for (package in azbatchenv$packages) {
-  library(package, character.only = TRUE)
-}
-
 parent.env(azbatchenv$exportenv) <- globalenv()
 
 enableCloudCombine <- azbatchenv$enableCloudCombine
@@ -88,7 +84,7 @@ if (typeof(cloudCombine) == "list" && enableCloudCombine) {
                                                                  "batchJobId",
                                                                  "chunkSize",
                                                                  "errorHandling",
-                                                                 "isError")) %dopar% {
+                                                                 "isError")) %do% {
       task <- tryCatch({
         readRDS(files[i])
       }, error = function(e) {

tests/testthat/utility.R

@@ -1,6 +1,6 @@
-getSettings <- function(dedicatedMin = 2,
+getSettings <- function(dedicatedMin = 0,
                         dedicatedMax = 2,
-                        lowPriorityMin = 2,
+                        lowPriorityMin = 0,
                         lowPriorityMax = 2,
                         poolName = "test-pool"){
   list(