Fix: Test Coverage on Azure Pipeline CI (#348)

* Added build.sh

* Fixed tests

* Added pipeline yml

* Fixed pipeline

* Removed

* Passing in args to script

* Added container image

* Fixed args

* Fixed args

* Fixed args

* Removed last script

* Added test management

* Added job dependency

* Moved package installation to build.sh

* fixed

* Added job dependency

* Added RProfile site

* Fixed REnvironment

* Added Rprofile

* Added ~/

* Fixing tests

* fixed method name

* Renaming samples & tests

* renamed more tests

* Fixed lint

* Removed other tests

* Fixed directory call for testthat

* Added typo and remove sample

* Fixed pipeline pointer

.vsts/pipeline.yml

@@ -0,0 +1,41 @@
+name: $(Build.SourceBranch)$(Rev:.r)
+
+trigger:
+  - master
+
+resources:
+  containers:
+  - container: linux
+    image: ubuntu:16.04
+
+jobs:
+- job: Build
+  displayName: Build Job
+  condition: succeeded()
+  pool:
+    vmImage: 'ubuntu-16.04'
+  steps:
+  - task: ShellScript@2
+    displayName: Build
+    inputs:
+      scriptPath: 'tests/test_scripts/build.sh'
+
+  - script: |
+      touch ~/.Rprofile
+      echo "Sys.setenv(BATCH_ACCOUNT_NAME ='"$(BATCH_ACCOUNT_NAME)"');" >> ~/.Rprofile
+      echo "Sys.setenv(BATCH_ACCOUNT_KEY ='"$(BATCH_ACCOUNT_KEY)"');" >> ~/.Rprofile
+      echo "Sys.setenv(BATCH_ACCOUNT_URL ='"$(BATCH_ACCOUNT_URL)"');" >> ~/.Rprofile
+      echo "Sys.setenv(STORAGE_ACCOUNT_NAME ='"$(STORAGE_ACCOUNT_NAME)"');" >> ~/.Rprofile
+      echo "Sys.setenv(STORAGE_ACCOUNT_KEY ='"$(STORAGE_ACCOUNT_KEY)"');" >> ~/.Rprofile
+      sudo R \
+        -e "getwd()" \
+        -e "devtools::install()" \
+        -e "devtools::build()" \
+        -e "doAzureParallel::generateCredentialsConfig('test_credentials.json', batchAccountName = Sys.getenv('BATCH_ACCOUNT_NAME'), batchAccountKey = Sys.getenv('BATCH_ACCOUNT_KEY'), batchAccountUrl = Sys.getenv('BATCH_ACCOUNT_URL'), storageAccountName = Sys.getenv('STORAGE_ACCOUNT_NAME'),  storageAccountKey = Sys.getenv('STORAGE_ACCOUNT_KEY'))"
+    condition: succeeded()
+    displayName: Create R Profile Environment Setting
+
+  - task: ShellScript@2
+    displayName: Run Unit Tests
+    inputs:
+      scriptPath: 'tests/testthat/unit_tests/unit_tests.sh'

samples/caret/caret_example.R

@@ -17,10 +17,10 @@ generateCredentialsConfig("credentials.json")
 setCredentials("credentials.json")
 
 # generate cluster config json file
-generateClusterConfig("cluster-caret.json")
+generateClusterConfig("caret_cluster.json")
 
 # Creating an Azure parallel backend
-cluster <- makeCluster(cluster = "cluster-caret.json")
+cluster <- makeCluster("caret_cluster.json")
 
 # Register your Azure parallel backend to the foreach implementation
 registerDoAzureParallel(cluster)
@@ -33,18 +33,18 @@ registerDoAzureParallel(cluster)
 # https://topepo.github.io/caret/index.html
 library(caret)
 
-# Set your chunk size of your tasks to 8 
+# Set the chunk size of your tasks to 8
 # So that caret knows in group tasks into larger chunks
 setChunkSize(8)
 
 # install DAAG to download the dataset 'spam7'
 install.packages("DAAG")
 library(DAAG)
 
-# 'spam7' is a data set that consists of 4601 email items, 
-# of which 1813 items were identified as spam. This sample 
-# has 7 features, one of which is titled 'yesno'. In this 
-# example, we will be classifying our data into 'yesno' to 
+# 'spam7' is a data set that consists of 4601 email items,
+# of which 1813 items were identified as spam. This sample
+# has 7 features, one of which is titled 'yesno'. In this
+# example, we will be classifying our data into 'yesno' to
 # identify which rows are spam, and which are not.
 
 # split the data into training and testing
@@ -53,7 +53,7 @@ inTraining <- createDataPartition(spam7$yesno, p = .75, list = FALSE)
 training <- spam7[ inTraining,]
 testing  <- spam7[-inTraining,]
 
-# Define the settings for the cv. Because we have already 
+# Define the settings for the cv. Because we have already
 # registered our parallel backend, Caret will know to use it
 fitControl <- trainControl(## 10-fold cross validation
                            method = "repeatedcv",
@@ -68,9 +68,9 @@ fitControl <- trainControl(## 10-fold cross validation
 
 
 rf_fit <- train(## classification column
-                 yesno ~ ., 
+                 yesno ~ .,
                  ## dataframe to train on
-                 data = training, 
+                 data = training,
                  ## model to use - other models are also available (see caret documentation)
                  method = "rf",
                  ## the metric to use for evaluation

samples/package_management/bioconductor/package_management_cluster.json → samples/package_management/bioconductor/bioconductor_cluster.json

@@ -1,5 +1,5 @@
 {
-  "name": "package_management",
+  "name": "bioconductor_pool",
   "vmSize": "Standard_A2_v2",
   "maxTasksPerNode": 1,
   "poolSize": {

tests/test_scripts/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+sudo echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" | sudo tee -a /etc/apt/sources.list
+
+gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
+gpg -a --export E084DAB9 | sudo apt-key add -
+
+sudo apt-get update
+sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
+sudo apt-get install -y libssl-dev libxml2-dev libgdal-dev libproj-dev libgsl-dev
+
+sudo R \
+  -e "getwd();" \
+  -e "install.packages(c('devtools', 'remotes', 'testthat', 'roxygen2'));"

tests/testthat/core/test-cluster.R

@@ -0,0 +1,81 @@
+context("Cluster Management Test")
+
+test_that("Create Cluster Test", {
+  testthat::skip_on_travis()
+  source("utility.R")
+
+  settings <- getSettings()
+  cluster <-
+    doAzureParallel::makeCluster(settings$clusterConfig, wait = FALSE)
+
+  cluster <- getCluster(cluster$poolId)
+  clusterList <- getClusterList()
+  filter <- list()
+  filter$state <- c("active", "deleting")
+
+  testthat::expect_true('test-pool' %in% clusterList$Id)
+})
+
+test_that("Get Cluster Test", {
+  testthat::skip_on_travis()
+  source("utility.R")
+
+  settings <- getSettings()
+
+  cluster <-
+    doAzureParallel::makeCluster(settings$clusterConfig, wait = FALSE)
+
+  cluster <- getCluster(cluster$poolId)
+  clusterList <- getClusterList()
+  filter <- list()
+  filter$state <- c("active", "deleting")
+
+  testthat::expect_true('test-pool' %in% clusterList$Id)
+
+  clusterList <- getClusterList(filter)
+
+  for (i in 1:length(clusterList$State)) {
+    testthat::expect_true(clusterList$State[i] == 'active' ||
+                          clusterList$State[i] == 'deleting')
+  }
+})
+
+test_that("Autoscale Cluster Test", {
+  testthat::skip_on_travis()
+  source("utility.R")
+
+  settings <- getSettings()
+
+  cluster <-
+    doAzureParallel::makeCluster(settings$clusterConfig, wait = FALSE)
+
+  cluster <- getCluster(cluster$poolId)
+  clusterList <- getClusterList()
+  filter <- list()
+  filter$state <- c("active", "deleting")
+
+  testthat::expect_true('test-pool' %in% clusterList$Id)
+
+  clusterList <- getClusterList(filter)
+
+  for (i in 1:length(clusterList$State)) {
+    testthat::expect_true(clusterList$State[i] == 'active' ||
+                            clusterList$State[i] == 'deleting')
+  }
+})
+
+test_that("Delete Cluster Test", {
+  testthat::skip_on_travis()
+  source("utility.R")
+
+  settings <- getSettings()
+
+  cluster <-
+    doAzureParallel::makeCluster(settings$clusterConfig, wait = FALSE)
+
+  doAzureParallel::stopCluster(cluster)
+
+  testthat::expect_true('test-pool' %in% clusterList$Id)
+
+  clusterList <- getClusterList(filter)
+})

tests/testthat/test-error-handling.R → tests/testthat/integration_tests/test-error-handling.R

@@ -4,9 +4,6 @@ test_that("Remove error handling with combine test", {
   source("utility.R")
   settings <- getSettings()
 
-  # set your credentials
-  doAzureParallel::setCredentials(settings$credentials)
-
   cluster <- doAzureParallel::makeCluster(settings$clusterConfig)
   doAzureParallel::registerDoAzureParallel(cluster)
 
@@ -31,9 +28,6 @@ test_that("Remove error handling test", {
   source("utility.R")
   settings <- getSettings()
 
-  # set your credentials
-  doAzureParallel::setCredentials(settings$credentials)
-
   settings$clusterConfig$poolId <- "error-handling-test"
   cluster <- doAzureParallel::makeCluster(settings$clusterConfig)
   doAzureParallel::registerDoAzureParallel(cluster)
@@ -58,9 +52,6 @@ test_that("Pass error handling test", {
   source("utility.R")
   settings <- getSettings()
 
-  # set your credentials
-  doAzureParallel::setCredentials(settings$credentials)
-
   settings$clusterConfig$poolId <- "error-handling-test"
   cluster <- doAzureParallel::makeCluster(settings$clusterConfig)
   doAzureParallel::registerDoAzureParallel(cluster)
@@ -82,14 +73,10 @@ test_that("Pass error handling test", {
 })
 
 test_that("Stop error handling test", {
-  testthat::skip("Manual Test")
   testthat::skip_on_travis()
   source("utility.R")
   settings <- getSettings()
 
-  # set your credentials
-  doAzureParallel::setCredentials(settings$credentials)
-
   settings$clusterConfig$poolId <- "error-handling-test"
   cluster <- doAzureParallel::makeCluster(settings$clusterConfig)
   doAzureParallel::registerDoAzureParallel(cluster)
@@ -99,11 +86,7 @@ test_that("Stop error handling test", {
   testthat::expect_error(
     res <-
       foreach::foreach(i = 1:4, .errorhandling = "stop") %dopar% {
-        if (i == 2) {
-          randomObject
-        }
-
-        i
+        randomObject
       }
   )
 })

tests/testthat/test-foreach-options.R → tests/testthat/integration_tests/test-foreach.R

@@ -1,11 +1,30 @@
-context("foreach options test")
-test_that("chunksize", {
+context("Integration Test")
+
+# Run this test for users to make sure the core features
+# of doAzureParallel are still working
+test_that("simple foreach 1 to 4", {
   testthat::skip_on_travis()
   source("utility.R")
   settings <- getSettings()
+  doAzureParallel::registerDoAzureParallel(cluster)
+
+  '%dopar%' <- foreach::'%dopar%'
+  res <-
+    foreach::foreach(i = 1:4) %dopar% {
+      i
+    }
+
+  res <- unname(res)
 
-  # set your credentials
-  doAzureParallel::setCredentials(settings$credentials)
+  testthat::expect_equal(length(res), 4)
+  testthat::expect_equal(res, list(1, 2, 3, 4))
+})
+
+context("Foreach Options Integration Test")
+test_that("chunksize", {
+  testthat::skip_on_travis()
+  source("utility.R")
+  settings <- getSettings()
 
   cluster <- doAzureParallel::makeCluster(settings$clusterConfig)
   doAzureParallel::registerDoAzureParallel(cluster)

tests/testthat/test-local-merge.R → tests/testthat/integration_tests/test-local-merge.R

@@ -5,11 +5,41 @@ test_that("merge job result locally test", {
   testthat::skip_on_travis()
   testthat::skip("Skipping merge job locally")
   source("utility.R")
-  settings <- gettingSettings()
   settings <- getSettings()
 
-  # set your credentials
-  doAzureParallel::setCredentials(settings$credentials)
+  cluster <- doAzureParallel::makeCluster(settings$clusterConfig)
+  doAzureParallel::registerDoAzureParallel(cluster)
+
+  setChunkSize(2)
+  '%dopar%' <- foreach::'%dopar%'
+  jobId <-
+    foreach::foreach(
+      i = 1:11,
+      .errorhandling = "pass",
+      .options.azure = list(
+        enableCloudCombine = FALSE,
+        wait = FALSE
+      )
+    ) %dopar% {
+      i
+    }
+
+  res <- getJobResult(jobId)
+
+  testthat::expect_equal(length(res),
+                         10)
+
+  for (i in 1:10) {
+    testthat::expect_equal(res[[i]],
+                           i)
+  }
+})
+
+test_that("merge job result locally test", {
+  testthat::skip_on_travis()
+  testthat::skip("Skipping merge job locally")
+  source("utility.R")
+  settings <- getSettings()
 
   cluster <- doAzureParallel::makeCluster(settings$clusterConfig)
   doAzureParallel::registerDoAzureParallel(cluster)