Bundling worker scripts into zip (#212)

* Added zip file extraction

* Fixed cluster setup

* Added cluster script and quiet zip function

* Replaced url name with correct zip file name

* Removed startup folder name

* Added apt-get install on job prep

* Fixed branch names

R/cluster.R

@@ -139,6 +139,7 @@ makeCluster <-
     config$containerImage <- dockerImage
     installAndStartContainerCommand <- "cluster_setup.sh"
 
+    # Note: Revert it to master once PR is approved
     dockerInstallCommand <- c(
       paste0(
         "wget https://raw.githubusercontent.com/Azure/doAzureParallel/",

R/doAzureParallel.R

@@ -358,7 +358,7 @@ setHttpTraffic <- function(value = FALSE) {
 
   retryCounter <- 0
   maxRetryCount <- 5
-  startupFolderName <- "startup"
+
   repeat {
     if (retryCounter > maxRetryCount) {
       stop(
@@ -395,23 +395,17 @@ setHttpTraffic <- function(value = FALSE) {
       stop(paste0("Error creating job: ", containerContent$message$value))
     }
 
+    nodeScriptsDir <- system.file("startup", package = "doAzureParallel")
+    nodeScriptsFiles <- list.files(nodeScriptsDir, full.names = TRUE)
+
+    nodeScriptsZip <- "node_scripts.zip"
+    # Zip Flags: Keeping console output clean and removing junk paths
+    utils::zip(nodeScriptsZip, files = nodeScriptsFiles, extras = "-j -q")
+
     # Uploading common job files for the worker node
     rAzureBatch::uploadBlob(id,
-                            system.file(startupFolderName, "worker.R", package = "doAzureParallel"))
-    rAzureBatch::uploadBlob(id,
-                            system.file(startupFolderName, "merger.R", package = "doAzureParallel"))
-    rAzureBatch::uploadBlob(
-      id,
-      system.file(startupFolderName, "install_github.R", package = "doAzureParallel")
-    )
-    rAzureBatch::uploadBlob(
-      id,
-      system.file(startupFolderName, "install_cran.R", package = "doAzureParallel")
-    )
-    rAzureBatch::uploadBlob(
-      id,
-      system.file(startupFolderName, "install_bioconductor.R", package = "doAzureParallel")
-    )
+                            nodeScriptsZip)
+    file.remove(nodeScriptsZip)
 
     # Creating common job environment for all tasks
     jobFileName <- paste0(id, ".rds")
@@ -421,28 +415,13 @@ setHttpTraffic <- function(value = FALSE) {
 
     # Creating read-only SAS token blob resource file urls
     sasToken <- rAzureBatch::createSasToken("r", "c", id)
-    workerScriptUrl <-
-      rAzureBatch::createBlobUrl(storageCredentials$name, id, "worker.R", sasToken)
-    mergerScriptUrl <-
-      rAzureBatch::createBlobUrl(storageCredentials$name, id, "merger.R", sasToken)
-    installGithubScriptUrl <-
-      rAzureBatch::createBlobUrl(storageCredentials$name,
-                                 id,
-                                 "install_github.R",
-                                 sasToken)
-    installCranScriptUrl <-
-      rAzureBatch::createBlobUrl(storageCredentials$name, id, "install_cran.R", sasToken)
-    installBioConductorScriptUrl <-
-      rAzureBatch::createBlobUrl(storageCredentials$name, id, "install_bioconductor.R", sasToken)
+    nodeScriptsZipUrl <-
+      rAzureBatch::createBlobUrl(storageCredentials$name, id, nodeScriptsZip, sasToken)
     jobCommonFileUrl <-
       rAzureBatch::createBlobUrl(storageCredentials$name, id, jobFileName, sasToken)
 
     requiredJobResourceFiles <- list(
-      rAzureBatch::createResourceFile(url = workerScriptUrl, fileName = "worker.R"),
-      rAzureBatch::createResourceFile(url = mergerScriptUrl, fileName = "merger.R"),
-      rAzureBatch::createResourceFile(url = installGithubScriptUrl, fileName = "install_github.R"),
-      rAzureBatch::createResourceFile(url = installCranScriptUrl, fileName = "install_cran.R"),
-      rAzureBatch::createResourceFile(url = installBioConductorScriptUrl, fileName = "install_bioconductor.R"),
+      rAzureBatch::createResourceFile(url = nodeScriptsZipUrl, fileName = nodeScriptsZip),
       rAzureBatch::createResourceFile(url = jobCommonFileUrl, fileName = jobFileName)
     )
 
@@ -495,7 +474,6 @@ setHttpTraffic <- function(value = FALSE) {
     }
   }
 
-
   job <- rAzureBatch::getJob(id)
 
   printJobInformation(

R/helpers.R

@@ -148,7 +148,13 @@
   poolInfo <- list("poolId" = poolId)
 
   # Default command for job preparation task
-  commands <- c("ls")
+  # Supports backwards compatibility if zip packages are missing, it will be installed
+  # Eventually, apt-get install command will be deprecated
+  commands <- c(
+    "apt-get -y install zip unzip",
+    "unzip -j $AZ_BATCH_JOB_PREP_WORKING_DIR/node_scripts.zip"
+  )
+
   if (!is.null(packages)) {
     jobPackages <-
       dockerRunCommand(containerImage,

inst/startup/cluster_setup.sh

@@ -16,6 +16,9 @@ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(
 apt-get -y update
 apt-get -y install docker-ce
 
+# Unzip resource files and set permissions
+apt-get -y install zip unzip
+
 # Check docker is running
 docker info > /dev/null 2>&1
 if [ $? -ne 0 ]; then