Fix: Do not use task chunk size during cloud combine function (#152)

* Fix task counts when group of tasks are not equal

* Added chunksize test

* Fixed pass case

Author: brnleehng

inst/startup/merger.R

@@ -31,7 +31,7 @@ enableCloudCombine <- azbatchenv$enableCloudCombine
 cloudCombine <- azbatchenv$cloudCombine
 
 if (typeof(cloudCombine) == "list" && enableCloudCombine) {
-  results <- vector("list", batchTasksCount * chunkSize)
+  results <- vector("list", batchTasksCount)
   count <- 1
 
   status <- tryCatch({
@@ -48,7 +48,7 @@ if (typeof(cloudCombine) == "list" && enableCloudCombine) {
                    "https://github.com/Azure/doAzureParallel/blob/master/docs/40-troubleshooting.md"))
       }
 
-      results <- vector("list", length(files) * chunkSize)
+      results <- vector("list", length(files))
 
       for (i in 1:length(files)) {
         task <- readRDS(files[i])
@@ -62,7 +62,7 @@ if (typeof(cloudCombine) == "list" && enableCloudCombine) {
           }
         }
 
-        for (t in 1:length(chunkSize)) {
+        for (t in 1:length(task)) {
           results[count] <- task[t]
           count <- count + 1
         }
@@ -84,7 +84,7 @@ if (typeof(cloudCombine) == "list" && enableCloudCombine) {
 
         if (file.exists(taskResult)) {
           task <- readRDS(taskResult)
-          for (t in 1:length(chunkSize)) {
+          for (t in 1:length(task)) {
             results[count] <- task[t]
             count <- count + 1
           }

tests/testthat/test-live.R

@@ -27,3 +27,32 @@ test_that("Scenario Test", {
   testthat::expect_equal(res,
                          list(2, 2, 2, 2))
 })
+
+test_that("Chunksize Test", {
+  testthat::skip_on_travis()
+  credentialsFileName <- "credentials.json"
+  clusterFileName <- "cluster.json"
+
+  doAzureParallel::generateCredentialsConfig(credentialsFileName)
+  doAzureParallel::generateClusterConfig(clusterFileName)
+
+  doAzureParallel::setCredentials(credentialsFileName)
+  cluster <- doAzureParallel::makeCluster(clusterFileName)
+  doAzureParallel::registerDoAzureParallel(cluster)
+
+  '%dopar%' <- foreach::'%dopar%'
+  res <-
+    foreach::foreach(i = 1:10, .options.azure = list(chunkSize = 3)) %dopar% {
+      i
+    }
+
+  doAzureParallel::stopCluster(cluster)
+
+  testthat::expect_equal(length(res),
+                         10)
+
+  for (i in 1:10) {
+    testthat::expect_equal(res[[i]],
+                           i)
+  }
+})