From 318ecc876c64b7085df46618278861c2ea9ff422 Mon Sep 17 00:00:00 2001 From: Felix Cheung Date: Fri, 27 Jan 2017 11:03:15 -0800 Subject: [PATCH 1/3] make sure Spark is installed --- R/pkg/inst/tests/testthat/test_Serde.R | 3 +++ R/pkg/inst/tests/testthat/test_binaryFile.R | 3 +++ R/pkg/inst/tests/testthat/test_binary_function.R | 3 +++ R/pkg/inst/tests/testthat/test_broadcast.R | 3 +++ R/pkg/inst/tests/testthat/test_context.R | 3 +++ R/pkg/inst/tests/testthat/test_includePackage.R | 3 +++ R/pkg/inst/tests/testthat/test_jvm_api.R | 3 +++ R/pkg/inst/tests/testthat/test_mllib_classification.R | 3 +++ R/pkg/inst/tests/testthat/test_mllib_clustering.R | 3 +++ R/pkg/inst/tests/testthat/test_mllib_recommendation.R | 3 +++ R/pkg/inst/tests/testthat/test_mllib_regression.R | 3 +++ R/pkg/inst/tests/testthat/test_mllib_stat.R | 3 +++ R/pkg/inst/tests/testthat/test_mllib_tree.R | 3 +++ R/pkg/inst/tests/testthat/test_parallelize_collect.R | 3 +++ R/pkg/inst/tests/testthat/test_rdd.R | 3 +++ R/pkg/inst/tests/testthat/test_shuffle.R | 3 +++ R/pkg/inst/tests/testthat/test_sparkSQL.R | 3 +++ R/pkg/inst/tests/testthat/test_take.R | 3 +++ R/pkg/inst/tests/testthat/test_textFile.R | 3 +++ R/pkg/inst/tests/testthat/test_utils.R | 3 +++ R/pkg/vignettes/sparkr-vignettes.Rmd | 3 +++ 21 files changed, 63 insertions(+) diff --git a/R/pkg/inst/tests/testthat/test_Serde.R b/R/pkg/inst/tests/testthat/test_Serde.R index b5f6f1b54fa8..ae2d5279467a 100644 --- a/R/pkg/inst/tests/testthat/test_Serde.R +++ b/R/pkg/inst/tests/testthat/test_Serde.R @@ -17,6 +17,9 @@ context("SerDe functionality") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + sparkSession <- sparkR.session(enableHiveSupport = FALSE) test_that("SerDe of primitive types", { diff --git a/R/pkg/inst/tests/testthat/test_binaryFile.R b/R/pkg/inst/tests/testthat/test_binaryFile.R index b5c279e3156e..0ca1ae3c784a 100644 --- a/R/pkg/inst/tests/testthat/test_binaryFile.R +++ b/R/pkg/inst/tests/testthat/test_binaryFile.R @@ -17,6 +17,9 @@ context("functions on binary files") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_binary_function.R b/R/pkg/inst/tests/testthat/test_binary_function.R index 59cb2e620440..ca3d1088530d 100644 --- a/R/pkg/inst/tests/testthat/test_binary_function.R +++ b/R/pkg/inst/tests/testthat/test_binary_function.R @@ -17,6 +17,9 @@ context("binary functions") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_broadcast.R b/R/pkg/inst/tests/testthat/test_broadcast.R index 65f204d096f4..fe58ad5fbe23 100644 --- a/R/pkg/inst/tests/testthat/test_broadcast.R +++ b/R/pkg/inst/tests/testthat/test_broadcast.R @@ -17,6 +17,9 @@ context("broadcast variables") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index caca06933952..4a4214fe356d 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -17,6 +17,9 @@ context("test functions in sparkR.R") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + test_that("Check masked functions", { # Check that we are not masking any new function from base, stats, testthat unexpectedly # NOTE: We should avoid adding entries to *namesOfMaskedCompletely* as masked functions make it diff --git a/R/pkg/inst/tests/testthat/test_includePackage.R b/R/pkg/inst/tests/testthat/test_includePackage.R index 563ea298c2dd..73891250ca80 100644 --- a/R/pkg/inst/tests/testthat/test_includePackage.R +++ b/R/pkg/inst/tests/testthat/test_includePackage.R @@ -17,6 +17,9 @@ context("include R packages") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_jvm_api.R b/R/pkg/inst/tests/testthat/test_jvm_api.R index 7348c893d0af..1ce667a91741 100644 --- a/R/pkg/inst/tests/testthat/test_jvm_api.R +++ b/R/pkg/inst/tests/testthat/test_jvm_api.R @@ -17,6 +17,9 @@ context("JVM API") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + sparkSession <- sparkR.session(enableHiveSupport = FALSE) test_that("Create and call methods on object", { diff --git a/R/pkg/inst/tests/testthat/test_mllib_classification.R b/R/pkg/inst/tests/testthat/test_mllib_classification.R index 2e0dea321e7b..7dc7d9c512a0 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_classification.R +++ b/R/pkg/inst/tests/testthat/test_mllib_classification.R @@ -19,6 +19,9 @@ library(testthat) context("MLlib classification algorithms, except for tree-based algorithms") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Tests for MLlib classification algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_clustering.R b/R/pkg/inst/tests/testthat/test_mllib_clustering.R index 9de8362cde8f..b1ff222ae5dc 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_clustering.R +++ b/R/pkg/inst/tests/testthat/test_mllib_clustering.R @@ -19,6 +19,9 @@ library(testthat) context("MLlib clustering algorithms") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Tests for MLlib clustering algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R b/R/pkg/inst/tests/testthat/test_mllib_recommendation.R index 6b1040db9305..80a6b86d37f5 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R +++ b/R/pkg/inst/tests/testthat/test_mllib_recommendation.R @@ -19,6 +19,9 @@ library(testthat) context("MLlib recommendation algorithms") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Tests for MLlib recommendation algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_regression.R b/R/pkg/inst/tests/testthat/test_mllib_regression.R index c450a151713f..734912b5bce1 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_regression.R +++ b/R/pkg/inst/tests/testthat/test_mllib_regression.R @@ -19,6 +19,9 @@ library(testthat) context("MLlib regression algorithms, except for tree-based algorithms") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Tests for MLlib regression algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_stat.R b/R/pkg/inst/tests/testthat/test_mllib_stat.R index beb148e7702f..1bfb4d8ded12 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_stat.R +++ b/R/pkg/inst/tests/testthat/test_mllib_stat.R @@ -19,6 +19,9 @@ library(testthat) context("MLlib statistics algorithms") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Tests for MLlib statistics algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_tree.R b/R/pkg/inst/tests/testthat/test_mllib_tree.R index e6fda251ebea..a403beffe17b 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_tree.R +++ b/R/pkg/inst/tests/testthat/test_mllib_tree.R @@ -19,6 +19,9 @@ library(testthat) context("MLlib tree-based algorithms") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Tests for MLlib tree-based algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_parallelize_collect.R b/R/pkg/inst/tests/testthat/test_parallelize_collect.R index 55972e1ba469..bfea8f5d254e 100644 --- a/R/pkg/inst/tests/testthat/test_parallelize_collect.R +++ b/R/pkg/inst/tests/testthat/test_parallelize_collect.R @@ -17,6 +17,9 @@ context("parallelize() and collect()") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Mock data numVector <- c(-10:97) numList <- list(sqrt(1), sqrt(2), sqrt(3), 4 ** 10) diff --git a/R/pkg/inst/tests/testthat/test_rdd.R b/R/pkg/inst/tests/testthat/test_rdd.R index 2c41a6b075b4..e0b86af54ea6 100644 --- a/R/pkg/inst/tests/testthat/test_rdd.R +++ b/R/pkg/inst/tests/testthat/test_rdd.R @@ -17,6 +17,9 @@ context("basic RDD functions") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_shuffle.R b/R/pkg/inst/tests/testthat/test_shuffle.R index d38efab0fd1d..818bb0724c15 100644 --- a/R/pkg/inst/tests/testthat/test_shuffle.R +++ b/R/pkg/inst/tests/testthat/test_shuffle.R @@ -17,6 +17,9 @@ context("partitionBy, groupByKey, reduceByKey etc.") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R index aaa8fb498c85..9fbdc7d7d1b2 100644 --- a/R/pkg/inst/tests/testthat/test_sparkSQL.R +++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R @@ -19,6 +19,9 @@ library(testthat) context("SparkSQL functions") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Utility function for easily checking the values of a StructField checkStructField <- function(actual, expectedName, expectedType, expectedNullable) { expect_equal(class(actual), "structField") diff --git a/R/pkg/inst/tests/testthat/test_take.R b/R/pkg/inst/tests/testthat/test_take.R index aaa532856c3d..90a143f4b466 100644 --- a/R/pkg/inst/tests/testthat/test_take.R +++ b/R/pkg/inst/tests/testthat/test_take.R @@ -17,6 +17,9 @@ context("tests RDD function take()") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # Mock data numVector <- c(-10:97) numList <- list(sqrt(1), sqrt(2), sqrt(3), 4 ** 10) diff --git a/R/pkg/inst/tests/testthat/test_textFile.R b/R/pkg/inst/tests/testthat/test_textFile.R index 3b466066e939..a86804dbee54 100644 --- a/R/pkg/inst/tests/testthat/test_textFile.R +++ b/R/pkg/inst/tests/testthat/test_textFile.R @@ -17,6 +17,9 @@ context("the textFile() function") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_utils.R b/R/pkg/inst/tests/testthat/test_utils.R index c87524842876..3c9467ae1753 100644 --- a/R/pkg/inst/tests/testthat/test_utils.R +++ b/R/pkg/inst/tests/testthat/test_utils.R @@ -17,6 +17,9 @@ context("functions in utils.R") +# Ensure Spark is installed +sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") + # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 6f11c5c51676..09c3379b801b 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -27,6 +27,9 @@ library(SparkR) We use default settings in which it runs in local mode. It auto downloads Spark package in the background if no previous installation is found. For more details about setup, see [Spark Session](#SetupSparkSession). +```{r, include=FALSE} +SparkR:::sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +``` ```{r, message=FALSE, results="hide"} sparkR.session() ``` From f51f504acb3a64da27bf0bddbb156c68d62d89bb Mon Sep 17 00:00:00 2001 From: Felix Cheung Date: Fri, 27 Jan 2017 22:55:54 -0800 Subject: [PATCH 2/3] simplify --- R/pkg/R/sparkR.R | 11 ++++++----- R/pkg/inst/tests/testthat/test_Serde.R | 2 +- R/pkg/inst/tests/testthat/test_binaryFile.R | 2 +- R/pkg/inst/tests/testthat/test_binary_function.R | 2 +- R/pkg/inst/tests/testthat/test_broadcast.R | 2 +- R/pkg/inst/tests/testthat/test_context.R | 2 +- R/pkg/inst/tests/testthat/test_includePackage.R | 2 +- R/pkg/inst/tests/testthat/test_jvm_api.R | 2 +- R/pkg/inst/tests/testthat/test_mllib_classification.R | 2 +- R/pkg/inst/tests/testthat/test_mllib_clustering.R | 2 +- R/pkg/inst/tests/testthat/test_mllib_recommendation.R | 2 +- R/pkg/inst/tests/testthat/test_mllib_regression.R | 2 +- R/pkg/inst/tests/testthat/test_mllib_stat.R | 2 +- R/pkg/inst/tests/testthat/test_mllib_tree.R | 2 +- R/pkg/inst/tests/testthat/test_parallelize_collect.R | 2 +- R/pkg/inst/tests/testthat/test_rdd.R | 2 +- R/pkg/inst/tests/testthat/test_shuffle.R | 2 +- R/pkg/inst/tests/testthat/test_sparkSQL.R | 2 +- R/pkg/inst/tests/testthat/test_take.R | 2 +- R/pkg/inst/tests/testthat/test_textFile.R | 2 +- R/pkg/inst/tests/testthat/test_utils.R | 2 +- R/pkg/vignettes/sparkr-vignettes.Rmd | 2 +- 22 files changed, 27 insertions(+), 26 deletions(-) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index 870e76b7292f..20e0e8fc08fa 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -585,16 +585,17 @@ processSparkPackages <- function(packages) { # @param deployMode whether to deploy your driver on the worker nodes (cluster) # or locally as an external client (client). # @return NULL if no need to update sparkHome, and new sparkHome otherwise. -sparkCheckInstall <- function(sparkHome, master, deployMode) { +sparkCheckInstall <- function( + sparkHome = Sys.getenv("SPARK_HOME"), + master = "local", + deployMode = "") { if (!isSparkRShell()) { if (!is.na(file.info(sparkHome)$isdir)) { - msg <- paste0("Spark package found in SPARK_HOME: ", sparkHome) - message(msg) + message("Spark package found in SPARK_HOME: ", sparkHome) NULL } else { if (interactive() || isMasterLocal(master)) { - msg <- paste0("Spark not found in SPARK_HOME: ", sparkHome) - message(msg) + message("Spark not found in SPARK_HOME: ", sparkHome) packageLocalDir <- install.spark() packageLocalDir } else if (isClientMode(master) || deployMode == "client") { diff --git a/R/pkg/inst/tests/testthat/test_Serde.R b/R/pkg/inst/tests/testthat/test_Serde.R index ae2d5279467a..c5668560d399 100644 --- a/R/pkg/inst/tests/testthat/test_Serde.R +++ b/R/pkg/inst/tests/testthat/test_Serde.R @@ -18,7 +18,7 @@ context("SerDe functionality") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_binaryFile.R b/R/pkg/inst/tests/testthat/test_binaryFile.R index 0ca1ae3c784a..6951a5f0e0f3 100644 --- a/R/pkg/inst/tests/testthat/test_binaryFile.R +++ b/R/pkg/inst/tests/testthat/test_binaryFile.R @@ -18,7 +18,7 @@ context("functions on binary files") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_binary_function.R b/R/pkg/inst/tests/testthat/test_binary_function.R index ca3d1088530d..dbbc42c2deeb 100644 --- a/R/pkg/inst/tests/testthat/test_binary_function.R +++ b/R/pkg/inst/tests/testthat/test_binary_function.R @@ -18,7 +18,7 @@ context("binary functions") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_broadcast.R b/R/pkg/inst/tests/testthat/test_broadcast.R index fe58ad5fbe23..aaaf78d6a10d 100644 --- a/R/pkg/inst/tests/testthat/test_broadcast.R +++ b/R/pkg/inst/tests/testthat/test_broadcast.R @@ -18,7 +18,7 @@ context("broadcast variables") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index 4a4214fe356d..a213de6212a5 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -18,7 +18,7 @@ context("test functions in sparkR.R") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() test_that("Check masked functions", { # Check that we are not masking any new function from base, stats, testthat unexpectedly diff --git a/R/pkg/inst/tests/testthat/test_includePackage.R b/R/pkg/inst/tests/testthat/test_includePackage.R index 73891250ca80..d366c68a9538 100644 --- a/R/pkg/inst/tests/testthat/test_includePackage.R +++ b/R/pkg/inst/tests/testthat/test_includePackage.R @@ -18,7 +18,7 @@ context("include R packages") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_jvm_api.R b/R/pkg/inst/tests/testthat/test_jvm_api.R index 1ce667a91741..36921fa55cbc 100644 --- a/R/pkg/inst/tests/testthat/test_jvm_api.R +++ b/R/pkg/inst/tests/testthat/test_jvm_api.R @@ -18,7 +18,7 @@ context("JVM API") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_classification.R b/R/pkg/inst/tests/testthat/test_mllib_classification.R index 7dc7d9c512a0..8a6892519989 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_classification.R +++ b/R/pkg/inst/tests/testthat/test_mllib_classification.R @@ -20,7 +20,7 @@ library(testthat) context("MLlib classification algorithms, except for tree-based algorithms") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Tests for MLlib classification algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_clustering.R b/R/pkg/inst/tests/testthat/test_mllib_clustering.R index b1ff222ae5dc..c6ca5dc13dbc 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_clustering.R +++ b/R/pkg/inst/tests/testthat/test_mllib_clustering.R @@ -20,7 +20,7 @@ library(testthat) context("MLlib clustering algorithms") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Tests for MLlib clustering algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R b/R/pkg/inst/tests/testthat/test_mllib_recommendation.R index 80a6b86d37f5..b1bb96816b0a 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R +++ b/R/pkg/inst/tests/testthat/test_mllib_recommendation.R @@ -20,7 +20,7 @@ library(testthat) context("MLlib recommendation algorithms") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Tests for MLlib recommendation algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_regression.R b/R/pkg/inst/tests/testthat/test_mllib_regression.R index 734912b5bce1..e21e6ebdd817 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_regression.R +++ b/R/pkg/inst/tests/testthat/test_mllib_regression.R @@ -20,7 +20,7 @@ library(testthat) context("MLlib regression algorithms, except for tree-based algorithms") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Tests for MLlib regression algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_stat.R b/R/pkg/inst/tests/testthat/test_mllib_stat.R index 1bfb4d8ded12..5fab7f7afcf6 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_stat.R +++ b/R/pkg/inst/tests/testthat/test_mllib_stat.R @@ -20,7 +20,7 @@ library(testthat) context("MLlib statistics algorithms") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Tests for MLlib statistics algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_tree.R b/R/pkg/inst/tests/testthat/test_mllib_tree.R index a403beffe17b..d9e4b92dfe64 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_tree.R +++ b/R/pkg/inst/tests/testthat/test_mllib_tree.R @@ -20,7 +20,7 @@ library(testthat) context("MLlib tree-based algorithms") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Tests for MLlib tree-based algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_parallelize_collect.R b/R/pkg/inst/tests/testthat/test_parallelize_collect.R index bfea8f5d254e..87b0f6aeff54 100644 --- a/R/pkg/inst/tests/testthat/test_parallelize_collect.R +++ b/R/pkg/inst/tests/testthat/test_parallelize_collect.R @@ -18,7 +18,7 @@ context("parallelize() and collect()") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Mock data numVector <- c(-10:97) diff --git a/R/pkg/inst/tests/testthat/test_rdd.R b/R/pkg/inst/tests/testthat/test_rdd.R index e0b86af54ea6..5124e1a51929 100644 --- a/R/pkg/inst/tests/testthat/test_rdd.R +++ b/R/pkg/inst/tests/testthat/test_rdd.R @@ -18,7 +18,7 @@ context("basic RDD functions") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_shuffle.R b/R/pkg/inst/tests/testthat/test_shuffle.R index 818bb0724c15..e131481179b4 100644 --- a/R/pkg/inst/tests/testthat/test_shuffle.R +++ b/R/pkg/inst/tests/testthat/test_shuffle.R @@ -18,7 +18,7 @@ context("partitionBy, groupByKey, reduceByKey etc.") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R index 9fbdc7d7d1b2..060939ccd7d2 100644 --- a/R/pkg/inst/tests/testthat/test_sparkSQL.R +++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R @@ -20,7 +20,7 @@ library(testthat) context("SparkSQL functions") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Utility function for easily checking the values of a StructField checkStructField <- function(actual, expectedName, expectedType, expectedNullable) { diff --git a/R/pkg/inst/tests/testthat/test_take.R b/R/pkg/inst/tests/testthat/test_take.R index 90a143f4b466..6bdb67a08cfa 100644 --- a/R/pkg/inst/tests/testthat/test_take.R +++ b/R/pkg/inst/tests/testthat/test_take.R @@ -18,7 +18,7 @@ context("tests RDD function take()") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # Mock data numVector <- c(-10:97) diff --git a/R/pkg/inst/tests/testthat/test_textFile.R b/R/pkg/inst/tests/testthat/test_textFile.R index a86804dbee54..35e75cd078c0 100644 --- a/R/pkg/inst/tests/testthat/test_textFile.R +++ b/R/pkg/inst/tests/testthat/test_textFile.R @@ -18,7 +18,7 @@ context("the textFile() function") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_utils.R b/R/pkg/inst/tests/testthat/test_utils.R index 3c9467ae1753..986c907b3e91 100644 --- a/R/pkg/inst/tests/testthat/test_utils.R +++ b/R/pkg/inst/tests/testthat/test_utils.R @@ -18,7 +18,7 @@ context("functions in utils.R") # Ensure Spark is installed -sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +sparkCheckInstall() # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 09c3379b801b..e046f19e3644 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -28,7 +28,7 @@ library(SparkR) We use default settings in which it runs in local mode. It auto downloads Spark package in the background if no previous installation is found. For more details about setup, see [Spark Session](#SetupSparkSession). ```{r, include=FALSE} -SparkR:::sparkCheckInstall(Sys.getenv("SPARK_HOME"), "local", "") +SparkR:::sparkCheckInstall() ``` ```{r, message=FALSE, results="hide"} sparkR.session() From cd1394a8aab39b20c9c747b5ddfe5a003081510a Mon Sep 17 00:00:00 2001 From: Felix Cheung Date: Sun, 12 Feb 2017 16:11:18 -0800 Subject: [PATCH 3/3] change install.spark --- R/pkg/R/install.R | 16 +++++++++++++--- R/pkg/R/sparkR.R | 5 +---- R/pkg/inst/tests/testthat/test_Serde.R | 3 --- R/pkg/inst/tests/testthat/test_binaryFile.R | 3 --- R/pkg/inst/tests/testthat/test_binary_function.R | 3 --- R/pkg/inst/tests/testthat/test_broadcast.R | 3 --- R/pkg/inst/tests/testthat/test_context.R | 3 --- R/pkg/inst/tests/testthat/test_includePackage.R | 3 --- R/pkg/inst/tests/testthat/test_jvm_api.R | 3 --- .../tests/testthat/test_mllib_classification.R | 3 --- .../inst/tests/testthat/test_mllib_clustering.R | 3 --- .../tests/testthat/test_mllib_recommendation.R | 3 --- .../inst/tests/testthat/test_mllib_regression.R | 3 --- R/pkg/inst/tests/testthat/test_mllib_stat.R | 3 --- R/pkg/inst/tests/testthat/test_mllib_tree.R | 3 --- .../tests/testthat/test_parallelize_collect.R | 3 --- R/pkg/inst/tests/testthat/test_rdd.R | 3 --- R/pkg/inst/tests/testthat/test_shuffle.R | 3 --- R/pkg/inst/tests/testthat/test_sparkSQL.R | 3 --- R/pkg/inst/tests/testthat/test_take.R | 3 --- R/pkg/inst/tests/testthat/test_textFile.R | 3 --- R/pkg/inst/tests/testthat/test_utils.R | 3 --- R/pkg/tests/run-all.R | 3 +++ R/pkg/vignettes/sparkr-vignettes.Rmd | 2 +- 24 files changed, 18 insertions(+), 68 deletions(-) diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R index 72386e68de4b..4ca7aa664e02 100644 --- a/R/pkg/R/install.R +++ b/R/pkg/R/install.R @@ -21,9 +21,9 @@ #' Download and Install Apache Spark to a Local Directory #' #' \code{install.spark} downloads and installs Spark to a local directory if -#' it is not found. The Spark version we use is the same as the SparkR version. -#' Users can specify a desired Hadoop version, the remote mirror site, and -#' the directory where the package is installed locally. +#' it is not found. If SPARK_HOME is set in the environment, and that directory is found, that is +#' returned. The Spark version we use is the same as the SparkR version. Users can specify a desired +#' Hadoop version, the remote mirror site, and the directory where the package is installed locally. #' #' The full url of remote file is inferred from \code{mirrorUrl} and \code{hadoopVersion}. #' \code{mirrorUrl} specifies the remote path to a Spark folder. It is followed by a subfolder @@ -68,6 +68,16 @@ #' \href{http://spark.apache.org/downloads.html}{Apache Spark} install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL, localDir = NULL, overwrite = FALSE) { + sparkHome <- Sys.getenv("SPARK_HOME") + if (isSparkRShell()) { + stopifnot(nchar(sparkHome) > 0) + message("Spark is already running in sparkR shell.") + return(invisible(sparkHome)) + } else if (!is.na(file.info(sparkHome)$isdir)) { + message("Spark package found in SPARK_HOME: ", sparkHome) + return(invisible(sparkHome)) + } + version <- paste0("spark-", packageVersion("SparkR")) hadoopVersion <- tolower(hadoopVersion) hadoopVersionName <- hadoopVersionName(hadoopVersion) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index 20e0e8fc08fa..61773ed3ee8c 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -585,10 +585,7 @@ processSparkPackages <- function(packages) { # @param deployMode whether to deploy your driver on the worker nodes (cluster) # or locally as an external client (client). # @return NULL if no need to update sparkHome, and new sparkHome otherwise. -sparkCheckInstall <- function( - sparkHome = Sys.getenv("SPARK_HOME"), - master = "local", - deployMode = "") { +sparkCheckInstall <- function(sparkHome, master, deployMode) { if (!isSparkRShell()) { if (!is.na(file.info(sparkHome)$isdir)) { message("Spark package found in SPARK_HOME: ", sparkHome) diff --git a/R/pkg/inst/tests/testthat/test_Serde.R b/R/pkg/inst/tests/testthat/test_Serde.R index c5668560d399..b5f6f1b54fa8 100644 --- a/R/pkg/inst/tests/testthat/test_Serde.R +++ b/R/pkg/inst/tests/testthat/test_Serde.R @@ -17,9 +17,6 @@ context("SerDe functionality") -# Ensure Spark is installed -sparkCheckInstall() - sparkSession <- sparkR.session(enableHiveSupport = FALSE) test_that("SerDe of primitive types", { diff --git a/R/pkg/inst/tests/testthat/test_binaryFile.R b/R/pkg/inst/tests/testthat/test_binaryFile.R index 6951a5f0e0f3..b5c279e3156e 100644 --- a/R/pkg/inst/tests/testthat/test_binaryFile.R +++ b/R/pkg/inst/tests/testthat/test_binaryFile.R @@ -17,9 +17,6 @@ context("functions on binary files") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_binary_function.R b/R/pkg/inst/tests/testthat/test_binary_function.R index dbbc42c2deeb..59cb2e620440 100644 --- a/R/pkg/inst/tests/testthat/test_binary_function.R +++ b/R/pkg/inst/tests/testthat/test_binary_function.R @@ -17,9 +17,6 @@ context("binary functions") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_broadcast.R b/R/pkg/inst/tests/testthat/test_broadcast.R index aaaf78d6a10d..65f204d096f4 100644 --- a/R/pkg/inst/tests/testthat/test_broadcast.R +++ b/R/pkg/inst/tests/testthat/test_broadcast.R @@ -17,9 +17,6 @@ context("broadcast variables") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index a213de6212a5..caca06933952 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -17,9 +17,6 @@ context("test functions in sparkR.R") -# Ensure Spark is installed -sparkCheckInstall() - test_that("Check masked functions", { # Check that we are not masking any new function from base, stats, testthat unexpectedly # NOTE: We should avoid adding entries to *namesOfMaskedCompletely* as masked functions make it diff --git a/R/pkg/inst/tests/testthat/test_includePackage.R b/R/pkg/inst/tests/testthat/test_includePackage.R index d366c68a9538..563ea298c2dd 100644 --- a/R/pkg/inst/tests/testthat/test_includePackage.R +++ b/R/pkg/inst/tests/testthat/test_includePackage.R @@ -17,9 +17,6 @@ context("include R packages") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_jvm_api.R b/R/pkg/inst/tests/testthat/test_jvm_api.R index 36921fa55cbc..7348c893d0af 100644 --- a/R/pkg/inst/tests/testthat/test_jvm_api.R +++ b/R/pkg/inst/tests/testthat/test_jvm_api.R @@ -17,9 +17,6 @@ context("JVM API") -# Ensure Spark is installed -sparkCheckInstall() - sparkSession <- sparkR.session(enableHiveSupport = FALSE) test_that("Create and call methods on object", { diff --git a/R/pkg/inst/tests/testthat/test_mllib_classification.R b/R/pkg/inst/tests/testthat/test_mllib_classification.R index 8a6892519989..2e0dea321e7b 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_classification.R +++ b/R/pkg/inst/tests/testthat/test_mllib_classification.R @@ -19,9 +19,6 @@ library(testthat) context("MLlib classification algorithms, except for tree-based algorithms") -# Ensure Spark is installed -sparkCheckInstall() - # Tests for MLlib classification algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_clustering.R b/R/pkg/inst/tests/testthat/test_mllib_clustering.R index c6ca5dc13dbc..9de8362cde8f 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_clustering.R +++ b/R/pkg/inst/tests/testthat/test_mllib_clustering.R @@ -19,9 +19,6 @@ library(testthat) context("MLlib clustering algorithms") -# Ensure Spark is installed -sparkCheckInstall() - # Tests for MLlib clustering algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R b/R/pkg/inst/tests/testthat/test_mllib_recommendation.R index b1bb96816b0a..6b1040db9305 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R +++ b/R/pkg/inst/tests/testthat/test_mllib_recommendation.R @@ -19,9 +19,6 @@ library(testthat) context("MLlib recommendation algorithms") -# Ensure Spark is installed -sparkCheckInstall() - # Tests for MLlib recommendation algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_regression.R b/R/pkg/inst/tests/testthat/test_mllib_regression.R index e21e6ebdd817..c450a151713f 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_regression.R +++ b/R/pkg/inst/tests/testthat/test_mllib_regression.R @@ -19,9 +19,6 @@ library(testthat) context("MLlib regression algorithms, except for tree-based algorithms") -# Ensure Spark is installed -sparkCheckInstall() - # Tests for MLlib regression algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_stat.R b/R/pkg/inst/tests/testthat/test_mllib_stat.R index 5fab7f7afcf6..beb148e7702f 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_stat.R +++ b/R/pkg/inst/tests/testthat/test_mllib_stat.R @@ -19,9 +19,6 @@ library(testthat) context("MLlib statistics algorithms") -# Ensure Spark is installed -sparkCheckInstall() - # Tests for MLlib statistics algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_mllib_tree.R b/R/pkg/inst/tests/testthat/test_mllib_tree.R index d9e4b92dfe64..e6fda251ebea 100644 --- a/R/pkg/inst/tests/testthat/test_mllib_tree.R +++ b/R/pkg/inst/tests/testthat/test_mllib_tree.R @@ -19,9 +19,6 @@ library(testthat) context("MLlib tree-based algorithms") -# Ensure Spark is installed -sparkCheckInstall() - # Tests for MLlib tree-based algorithms in SparkR sparkSession <- sparkR.session(enableHiveSupport = FALSE) diff --git a/R/pkg/inst/tests/testthat/test_parallelize_collect.R b/R/pkg/inst/tests/testthat/test_parallelize_collect.R index 87b0f6aeff54..55972e1ba469 100644 --- a/R/pkg/inst/tests/testthat/test_parallelize_collect.R +++ b/R/pkg/inst/tests/testthat/test_parallelize_collect.R @@ -17,9 +17,6 @@ context("parallelize() and collect()") -# Ensure Spark is installed -sparkCheckInstall() - # Mock data numVector <- c(-10:97) numList <- list(sqrt(1), sqrt(2), sqrt(3), 4 ** 10) diff --git a/R/pkg/inst/tests/testthat/test_rdd.R b/R/pkg/inst/tests/testthat/test_rdd.R index 5124e1a51929..2c41a6b075b4 100644 --- a/R/pkg/inst/tests/testthat/test_rdd.R +++ b/R/pkg/inst/tests/testthat/test_rdd.R @@ -17,9 +17,6 @@ context("basic RDD functions") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_shuffle.R b/R/pkg/inst/tests/testthat/test_shuffle.R index e131481179b4..d38efab0fd1d 100644 --- a/R/pkg/inst/tests/testthat/test_shuffle.R +++ b/R/pkg/inst/tests/testthat/test_shuffle.R @@ -17,9 +17,6 @@ context("partitionBy, groupByKey, reduceByKey etc.") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R index 060939ccd7d2..aaa8fb498c85 100644 --- a/R/pkg/inst/tests/testthat/test_sparkSQL.R +++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R @@ -19,9 +19,6 @@ library(testthat) context("SparkSQL functions") -# Ensure Spark is installed -sparkCheckInstall() - # Utility function for easily checking the values of a StructField checkStructField <- function(actual, expectedName, expectedType, expectedNullable) { expect_equal(class(actual), "structField") diff --git a/R/pkg/inst/tests/testthat/test_take.R b/R/pkg/inst/tests/testthat/test_take.R index 6bdb67a08cfa..aaa532856c3d 100644 --- a/R/pkg/inst/tests/testthat/test_take.R +++ b/R/pkg/inst/tests/testthat/test_take.R @@ -17,9 +17,6 @@ context("tests RDD function take()") -# Ensure Spark is installed -sparkCheckInstall() - # Mock data numVector <- c(-10:97) numList <- list(sqrt(1), sqrt(2), sqrt(3), 4 ** 10) diff --git a/R/pkg/inst/tests/testthat/test_textFile.R b/R/pkg/inst/tests/testthat/test_textFile.R index 35e75cd078c0..3b466066e939 100644 --- a/R/pkg/inst/tests/testthat/test_textFile.R +++ b/R/pkg/inst/tests/testthat/test_textFile.R @@ -17,9 +17,6 @@ context("the textFile() function") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/inst/tests/testthat/test_utils.R b/R/pkg/inst/tests/testthat/test_utils.R index 986c907b3e91..c87524842876 100644 --- a/R/pkg/inst/tests/testthat/test_utils.R +++ b/R/pkg/inst/tests/testthat/test_utils.R @@ -17,9 +17,6 @@ context("functions in utils.R") -# Ensure Spark is installed -sparkCheckInstall() - # JavaSparkContext handle sparkSession <- sparkR.session(enableHiveSupport = FALSE) sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession) diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R index 1d04656ac259..ab8d1ca01994 100644 --- a/R/pkg/tests/run-all.R +++ b/R/pkg/tests/run-all.R @@ -21,4 +21,7 @@ library(SparkR) # Turn all warnings into errors options("warn" = 2) +# Setup global test environment +install.spark() + test_package("SparkR") diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index e046f19e3644..da25e6275906 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -28,7 +28,7 @@ library(SparkR) We use default settings in which it runs in local mode. It auto downloads Spark package in the background if no previous installation is found. For more details about setup, see [Spark Session](#SetupSparkSession). ```{r, include=FALSE} -SparkR:::sparkCheckInstall() +install.spark() ``` ```{r, message=FALSE, results="hide"} sparkR.session()