From b40fa457ceafe4467a374648cc42944285740c10 Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Wed, 24 Jun 2015 21:57:55 -0700 Subject: [PATCH 1/8] [SPARK-8603] [sparkR] Wrong file path computed in windows --- R/pkg/R/client.R | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index cf2e5ddeb7a9..51302fa87a9f 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -43,6 +43,20 @@ determineSparkSubmitBin <- function() { sparkSubmitBinName } +# R supports both file separator in the file path (Unix : / and Windows: \) irrespective of the operating system +# but when passing file path to Java or script program, it has to be converted according to operating system + +determinefileSeperator <- function() { + if (.Platform$OS.type == "unix") { + fileSeperator <- .Platform$file.sep + } else { +# .Platform$file.sep contains "/" for windows too +# http://www.inside-r.org/r-doc/base/file.path + fileSeperator <- "\\" + } + fileSeperator +} + generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { if (jars != "") { jars <- paste("--jars", jars) @@ -59,7 +73,8 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBin <- determineSparkSubmitBin() if (sparkHome != "") { - sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName) + fileSeperator <- determinefileSeperator() + sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fileSeperator) } else { sparkSubmitBin <- sparkSubmitBinName } From ecbde2e07405fd186e4d3aa30e140bab6079a8eb Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Wed, 24 Jun 2015 22:04:11 -0700 Subject: [PATCH 2/8] [SPARK-8603] [sparkR] In windows, Incorrect file seperator passed to Java and Scripts from R --- R/pkg/R/client.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 51302fa87a9f..40b354f3b4b6 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -52,7 +52,7 @@ determinefileSeperator <- function() { } else { # .Platform$file.sep contains "/" for windows too # http://www.inside-r.org/r-doc/base/file.path - fileSeperator <- "\\" + fileSeperator <- "\\" } fileSeperator } From a2b7ee8cbb05761dc0d472abfefe5db0e5ee856e Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Wed, 24 Jun 2015 22:16:30 -0700 Subject: [PATCH 3/8] [SPARK-8603] [sparkR] In windows, Incorrect file seperator passed to Java and Scripts from R --- R/pkg/R/client.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 40b354f3b4b6..8b9f3f5dfb64 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -73,8 +73,9 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBin <- determineSparkSubmitBin() if (sparkHome != "") { + fileSeperator <- determinefileSeperator() - sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fileSeperator) + sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBin, fileSeperator) } else { sparkSubmitBin <- sparkSubmitBinName } From ad748dc1a53c49bebf9cfc74f25ba269626618a8 Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Wed, 24 Jun 2015 22:19:01 -0700 Subject: [PATCH 4/8] [SPARK-8603] [sparkR] In windows, Incorrect file seperator passed to Java and Scripts from R --- R/pkg/R/client.R | 1 - 1 file changed, 1 deletion(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 8b9f3f5dfb64..7846bf8b3a6b 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -73,7 +73,6 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBin <- determineSparkSubmitBin() if (sparkHome != "") { - fileSeperator <- determinefileSeperator() sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBin, fileSeperator) } else { From fdfd5a87412b8b4ca4bacd6374d20cc4fc9a8775 Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Thu, 25 Jun 2015 00:15:57 -0700 Subject: [PATCH 5/8] [SPARK-8603] [sparkR] In windows, Incorrect file seperator passed to Java and Scripts from R --- R/pkg/R/client.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 7846bf8b3a6b..ffd0b27d601a 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -71,10 +71,10 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack } launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { - sparkSubmitBin <- determineSparkSubmitBin() + sparkSubmitBinName <- determineSparkSubmitBin() if (sparkHome != "") { fileSeperator <- determinefileSeperator() - sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBin, fileSeperator) + sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fileSeperator) } else { sparkSubmitBin <- sparkSubmitBinName } From bd80bcfe8bc746f8e666eb6a862e9f1b391b1c36 Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Thu, 25 Jun 2015 00:37:41 -0700 Subject: [PATCH 6/8] launchBackend function signature missmatch --- R/pkg/R/sparkR.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index 8f81d5640c1d..633b869f9178 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -132,7 +132,7 @@ sparkR.init <- function( sparkHome = sparkHome, jars = jars, sparkSubmitOpts = Sys.getenv("SPARKR_SUBMIT_ARGS", "sparkr-shell"), - sparkPackages = sparkPackages) + packages = sparkPackages) # wait atmost 100 seconds for JVM to launch wait <- 0.1 for (i in 1:25) { From a0147a988dec5b51a2398c587c94bc93470cd560 Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Thu, 25 Jun 2015 00:43:48 -0700 Subject: [PATCH 7/8] [SPARK-8603] [sparkR] In windows, Incorrect file seperator passed to Java and Scripts from R --- R/pkg/R/client.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index ffd0b27d601a..3d2c2088f7a1 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -74,7 +74,7 @@ launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBinName <- determineSparkSubmitBin() if (sparkHome != "") { fileSeperator <- determinefileSeperator() - sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fileSeperator) + sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fsep = fileSeperator) } else { sparkSubmitBin <- sparkSubmitBinName } From b981f96dcdca06eef5a5e854757108b05c85d445 Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Thu, 25 Jun 2015 10:59:42 -0700 Subject: [PATCH 8/8] launchBackend function signature missmatch --- R/pkg/R/sparkR.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index 633b869f9178..1bff7df49bea 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -132,7 +132,7 @@ sparkR.init <- function( sparkHome = sparkHome, jars = jars, sparkSubmitOpts = Sys.getenv("SPARKR_SUBMIT_ARGS", "sparkr-shell"), - packages = sparkPackages) + packages = sparkPackages ) # wait atmost 100 seconds for JVM to launch wait <- 0.1 for (i in 1:25) {