diff --git a/R/cluster.R b/R/cluster.R index f56f3676..5ef6d039 100644 --- a/R/cluster.R +++ b/R/cluster.R @@ -139,6 +139,7 @@ makeCluster <- config$containerImage <- dockerImage installAndStartContainerCommand <- "cluster_setup.sh" + # Note: Revert it to master once PR is approved dockerInstallCommand <- c( paste0( "wget https://raw.githubusercontent.com/Azure/doAzureParallel/", diff --git a/R/doAzureParallel.R b/R/doAzureParallel.R index 52f294ed..510af359 100644 --- a/R/doAzureParallel.R +++ b/R/doAzureParallel.R @@ -358,7 +358,7 @@ setHttpTraffic <- function(value = FALSE) { retryCounter <- 0 maxRetryCount <- 5 - startupFolderName <- "startup" + repeat { if (retryCounter > maxRetryCount) { stop( @@ -395,23 +395,17 @@ setHttpTraffic <- function(value = FALSE) { stop(paste0("Error creating job: ", containerContent$message$value)) } + nodeScriptsDir <- system.file("startup", package = "doAzureParallel") + nodeScriptsFiles <- list.files(nodeScriptsDir, full.names = TRUE) + + nodeScriptsZip <- "node_scripts.zip" + # Zip Flags: Keeping console output clean and removing junk paths + utils::zip(nodeScriptsZip, files = nodeScriptsFiles, extras = "-j -q") + # Uploading common job files for the worker node rAzureBatch::uploadBlob(id, - system.file(startupFolderName, "worker.R", package = "doAzureParallel")) - rAzureBatch::uploadBlob(id, - system.file(startupFolderName, "merger.R", package = "doAzureParallel")) - rAzureBatch::uploadBlob( - id, - system.file(startupFolderName, "install_github.R", package = "doAzureParallel") - ) - rAzureBatch::uploadBlob( - id, - system.file(startupFolderName, "install_cran.R", package = "doAzureParallel") - ) - rAzureBatch::uploadBlob( - id, - system.file(startupFolderName, "install_bioconductor.R", package = "doAzureParallel") - ) + nodeScriptsZip) + file.remove(nodeScriptsZip) # Creating common job environment for all tasks jobFileName <- paste0(id, ".rds") @@ -421,28 +415,13 @@ setHttpTraffic <- function(value = FALSE) { # Creating read-only SAS token blob resource file urls sasToken <- rAzureBatch::createSasToken("r", "c", id) - workerScriptUrl <- - rAzureBatch::createBlobUrl(storageCredentials$name, id, "worker.R", sasToken) - mergerScriptUrl <- - rAzureBatch::createBlobUrl(storageCredentials$name, id, "merger.R", sasToken) - installGithubScriptUrl <- - rAzureBatch::createBlobUrl(storageCredentials$name, - id, - "install_github.R", - sasToken) - installCranScriptUrl <- - rAzureBatch::createBlobUrl(storageCredentials$name, id, "install_cran.R", sasToken) - installBioConductorScriptUrl <- - rAzureBatch::createBlobUrl(storageCredentials$name, id, "install_bioconductor.R", sasToken) + nodeScriptsZipUrl <- + rAzureBatch::createBlobUrl(storageCredentials$name, id, nodeScriptsZip, sasToken) jobCommonFileUrl <- rAzureBatch::createBlobUrl(storageCredentials$name, id, jobFileName, sasToken) requiredJobResourceFiles <- list( - rAzureBatch::createResourceFile(url = workerScriptUrl, fileName = "worker.R"), - rAzureBatch::createResourceFile(url = mergerScriptUrl, fileName = "merger.R"), - rAzureBatch::createResourceFile(url = installGithubScriptUrl, fileName = "install_github.R"), - rAzureBatch::createResourceFile(url = installCranScriptUrl, fileName = "install_cran.R"), - rAzureBatch::createResourceFile(url = installBioConductorScriptUrl, fileName = "install_bioconductor.R"), + rAzureBatch::createResourceFile(url = nodeScriptsZipUrl, fileName = nodeScriptsZip), rAzureBatch::createResourceFile(url = jobCommonFileUrl, fileName = jobFileName) ) @@ -495,7 +474,6 @@ setHttpTraffic <- function(value = FALSE) { } } - job <- rAzureBatch::getJob(id) printJobInformation( diff --git a/R/helpers.R b/R/helpers.R index a740b778..d1af4526 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -148,7 +148,13 @@ poolInfo <- list("poolId" = poolId) # Default command for job preparation task - commands <- c("ls") + # Supports backwards compatibility if zip packages are missing, it will be installed + # Eventually, apt-get install command will be deprecated + commands <- c( + "apt-get -y install zip unzip", + "unzip -j $AZ_BATCH_JOB_PREP_WORKING_DIR/node_scripts.zip" + ) + if (!is.null(packages)) { jobPackages <- dockerRunCommand(containerImage, diff --git a/inst/startup/cluster_setup.sh b/inst/startup/cluster_setup.sh index 4c38d2bf..d154eefa 100644 --- a/inst/startup/cluster_setup.sh +++ b/inst/startup/cluster_setup.sh @@ -16,6 +16,9 @@ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $( apt-get -y update apt-get -y install docker-ce +# Unzip resource files and set permissions +apt-get -y install zip unzip + # Check docker is running docker info > /dev/null 2>&1 if [ $? -ne 0 ]; then