From 887893377a3607a11690e58cc44740d551e0d490 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 4 Jan 2024 14:40:18 +0900 Subject: [PATCH] Separate pathes properly --- .../scala/org/apache/spark/api/python/PythonUtils.scala | 6 ++++-- .../spark/sql/execution/datasources/DataSourceManager.scala | 4 +--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala index 26c790a124470..929058fb7185d 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala @@ -36,7 +36,7 @@ private[spark] object PythonUtils extends Logging { val PY4J_ZIP_NAME = "py4j-0.10.9.7-src.zip" /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */ - def sparkPythonPath: String = { + def sparkPythonPaths: Seq[String] = { val pythonPath = new ArrayBuffer[String] for (sparkHome <- sys.env.get("SPARK_HOME")) { pythonPath += Seq(sparkHome, "python", "lib", "pyspark.zip").mkString(File.separator) @@ -44,9 +44,11 @@ private[spark] object PythonUtils extends Logging { Seq(sparkHome, "python", "lib", PY4J_ZIP_NAME).mkString(File.separator) } pythonPath ++= SparkContext.jarOfObject(this) - pythonPath.mkString(File.pathSeparator) + pythonPath.toSeq } + def sparkPythonPath: String = sparkPythonPaths.mkString(File.pathSeparator) + /** Merge PYTHONPATHS with the appropriate separator. Ignores blank strings. */ def mergePythonPaths(paths: String*): String = { paths.filter(_ != "").mkString(File.pathSeparator) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceManager.scala index 4fc636a59e5a3..236ab98969e5b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceManager.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceManager.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.execution.datasources import java.io.File import java.util.Locale import java.util.concurrent.ConcurrentHashMap -import java.util.regex.Pattern import scala.jdk.CollectionConverters._ @@ -91,8 +90,7 @@ object DataSourceManager extends Logging { private lazy val shouldLoadPythonDataSources: Boolean = { Utils.checkCommandAvailable(PythonUtils.defaultPythonExec) && // Make sure PySpark zipped files also exist. - PythonUtils.sparkPythonPath - .split(Pattern.quote(File.separator)).forall(new File(_).exists()) + PythonUtils.sparkPythonPaths.forall(new File(_).exists()) } private def initialDataSourceBuilders: Map[String, UserDefinedPythonDataSource] = {