rename jobAutoDelete to autoDeleteJob to workaround R bugs and update docs

Author: zfengms

NAMESPACE

@@ -17,10 +17,10 @@ export(listStorageFiles)
 export(makeCluster)
 export(registerDoAzureParallel)
 export(resizeCluster)
+export(setAutoDeleteJob)
 export(setChunkSize)
 export(setCredentials)
 export(setHttpTraffic)
-export(setJobAutoDelete)
 export(setReduce)
 export(setVerbose)
 export(stopCluster)

R/doAzureParallel.R

@@ -73,18 +73,18 @@ setChunkSize <- function(value = 1) {
   assign("chunkSize", value, envir = .doAzureBatchGlobals)
 }
 
-#' Specify whether to delete job/jobresult after asychronous job is completed.
+#' Specify whether to delete job and its result after asychronous job is completed.
 #'
 #' @param value boolean of TRUE or FALSE
 #'
 #' @examples
-#' setJobAutoDelete(FALSE)
+#' setAutoDeleteJob(FALSE)
 #' @export
-setJobAutoDelete <- function(value = TRUE) {
+setAutoDeleteJob <- function(value = TRUE) {
   if (!is.logical(value))
-    stop("setJobAutoDelete requires a boolean argument")
+    stop("setAutoDeleteJob requires a boolean argument")
 
-  assign("jobAutoDelete", value, envir = .doAzureBatchGlobals)
+  assign("autoDeleteJob", value, envir = .doAzureBatchGlobals)
 }
 
 #' Apply reduce function on a group of iterations of the foreach loop together per task.
@@ -247,15 +247,15 @@ setHttpTraffic <- function(value = FALSE) {
   }
 
   # by default, delete both job and job result after synchronous job is completed
-  jobAutoDelete <- TRUE
+  autoDeleteJob <- TRUE
 
-  if (exists("jobAutoDelete", envir = .doAzureBatchGlobals)) {
-    jobAutoDelete <- get("jobAutoDelete", envir = .doAzureBatchGlobals)
+  if (exists("autoDeleteJob", envir = .doAzureBatchGlobals)) {
+    autoDeleteJob <- get("autoDeleteJob", envir = .doAzureBatchGlobals)
   }
 
-  if (!is.null(obj$options$azure$jobAutoDelete) &&
-      is.logical(obj$options$azure$jobAutoDelete)) {
-    jobAutoDelete <- obj$options$azure$jobAutoDelete
+  if (!is.null(obj$options$azure$autoDeleteJob) &&
+      is.logical(obj$options$azure$autoDeleteJob)) {
+    autoDeleteJob <- obj$options$azure$autoDeleteJob
   }
 
   inputs <- FALSE
@@ -585,7 +585,7 @@ setHttpTraffic <- function(value = FALSE) {
 
           numberOfFailedTasks <- sum(unlist(failTasks))
 
-          if (numberOfFailedTasks > 0 && jobAutoDelete == FALSE) {
+          if (numberOfFailedTasks > 0 && autoDeleteJob == FALSE) {
             .createErrorViewerPane(id, failTasks)
           }
 
@@ -607,15 +607,18 @@ setHttpTraffic <- function(value = FALSE) {
               fill = TRUE)
 
           # delete job from batch service and job result from storage blob
-          if (jobAutoDelete) {
+          if (autoDeleteJob) {
             deleteJob(id)
           }
 
           if (identical(obj$errorHandling, "stop") &&
               !is.null(errorValue)) {
-            msg <- sprintf("task %d failed - '%s'",
-                           errorIndex,
-                           conditionMessage(errorValue))
+            msg <-
+              sprintf(
+                "task %d failed - '%s'.\r\nBy default job and its result is deleted after run is over, use setAutoDeleteJob(FALSE) or autoDeleteJob = FALSE option to keep them for investigation.",
+                errorIndex,
+                conditionMessage(errorValue)
+              )
             stop(simpleError(msg, call = expr))
           }
           else {

docs/23-persistent-storage.md

@@ -22,7 +22,7 @@ By default, *wait* is set to TRUE. This blocks the R session. By setting *wait*
 
 ## Getting results from storage
 
-When the user is ready to get their results in a new session, the user use the following command:
+When the user is ready to get their results in a new session, the user uses the following command:
 
 ```R
 my_job_id <- "my_unique_job_id"

docs/40-troubleshooting.md

@@ -67,7 +67,18 @@ This issue is due to certain compiler flags not available in the default version
 Since doAzureParallel uses Microsoft R Open version 3.3 as the default version of R, it will automatically try to pull package from [MRAN](https://mran.microsoft.com/) rather than CRAN. This is a big benefit when wanting to use a constant version of a package but does not always contain references to the latest versions. To use a specific version from CRAN or a different MRAN snapshot date, use the [command line](./30-customize-cluster.md#running-commands-when-the-cluster-starts) in the cluster configuration to manually install the packages you need.
 
 ## Viewing files from Azure Storage
-In every foreach run, the job will push its logs into Azure Storage that can be fetched by the user. For more information on reading log files, check out [managing storage](./41-managing-storage-via-R.md). 
+In every foreach run, the job will push its logs into Azure Storage that can be fetched by the user. For more information on reading log files, check out [managing storage](./41-managing-storage-via-R.md).
+
+By default, when wait is set to TRUE, job and its result is automatically deleted after the run is completed. To keep the job and its result for investigation purpose, you can set a global environment setting or specify an option in foreach loop to keep it.
+
+```R
+# This will set a global setting to keep job and its result after run is completed. 
+setAutoDeleteJob(FALSE)
+
+# This will keep job and its result at each job level after run is completed.
+options <- list(autoDeleteJob = FALSE)
+foreach::foreach(i = 1:4, .options.azure = opt) %dopar% { ... }
+```
 
 ## Viewing files directly from compute node
 Cluster setup logs are not persisted. `getClusterFile` function will fetch any files including stdout and stderr log files in the cluster. This is particularly useful for users that utilizing [customize script](./30-customize-cluster.md) on their nodes and installing specific [packages](./20-package-management.md).

docs/42-faq.md

@@ -19,4 +19,7 @@ Yes. The [command line](./30-customize-cluster.md#running-commands-when-the-clus
 No. doAzureParallel is built on top of the Linux CentOS distribution and will not work with Windows-specific packages.
 
 ## Why am I getting the error: could not find function "startsWith"?
-doAzureParallel requires you to run R 3.3 or greater on you local machine.
+doAzureParallel requires you to run R 3.3 or greater on you local machine.
+
+## My job failed but I can't find my job and its result?
+if you set wait = TRUE, job and its result is automatically deleted, to keep them for investigation purpose, you can set global option using setAutoDeleteJob(FALSE), or use autoDeleteJob option at foreach level.

man/setAutoDeleteJob.Rd

@@ -0,0 +1,67 @@
+# Run this test for users to make sure the autodeletejob feature
+# of doAzureParallel is still working
+context("auto delete job scenario test")
+test_that("auto delete job as foreach option test", {
+  testthat::skip("Live test")
+  testthat::skip_on_travis()
+  credentialsFileName <- "credentials.json"
+  clusterFileName <- "cluster.json"
+
+  doAzureParallel::generateCredentialsConfig(credentialsFileName)
+  doAzureParallel::generateClusterConfig(clusterFileName)
+
+  doAzureParallel::setCredentials(credentialsFileName)
+  cluster <- doAzureParallel::makeCluster(clusterFileName)
+  doAzureParallel::registerDoAzureParallel(cluster)
+
+  # use autoDeleteJob flag to keep the job and its result
+  '%dopar%' <- foreach::'%dopar%'
+  res <-
+    foreach::foreach(i = 1:10,
+                     .options.azure = list(autoDeleteJob = FALSE)) %dopar% {
+      i
+    }
+
+  testthat::expect_equal(length(res),
+                         10)
+
+  for (i in 1:10) {
+    testthat::expect_equal(res[[i]],
+                           i)
+  }
+
+  # find the job id from the output of above command and call deleteJob(jobId) when you no longer need the job and its result
+})
+
+test_that("auto delete job as global setting test", {
+  testthat::skip("Live test")
+  testthat::skip_on_travis()
+  credentialsFileName <- "credentials.json"
+  clusterFileName <- "cluster.json"
+
+  doAzureParallel::generateCredentialsConfig(credentialsFileName)
+  doAzureParallel::generateClusterConfig(clusterFileName)
+
+  doAzureParallel::setCredentials(credentialsFileName)
+  cluster <- doAzureParallel::makeCluster(clusterFileName)
+  doAzureParallel::registerDoAzureParallel(cluster)
+
+  # set autoDeleteJob flag to FALSE to keep the job and its result
+  setAutoDeleteJob(FALSE)
+
+  '%dopar%' <- foreach::'%dopar%'
+  res <-
+    foreach::foreach(i = 1:10) %dopar% {
+                       i
+                     }
+
+  testthat::expect_equal(length(res),
+                         10)
+
+  for (i in 1:10) {
+    testthat::expect_equal(res[[i]],
+                           i)
+  }
+
+  # find the job id from the output of above command and call deleteJob(jobId) when you no longer need the job and its result
+})