From 1aedfdf10e740e45d20e86ec93152b034a2346ed Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 4 Feb 2020 19:53:11 +0900 Subject: [PATCH] Rename spark.sql.pandas.udf.buffer.size to spark.sql.execution.pandas.udf.buffer.size --- python/pyspark/sql/tests/test_pandas_udf_scalar.py | 2 +- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py index 8e7e85ffe6b7..b07de3c0b3cc 100644 --- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py +++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py @@ -868,7 +868,7 @@ def test_close(batch_iter): with QuietTest(self.sc): with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": 1, - "spark.sql.pandas.udf.buffer.size": 4}): + "spark.sql.execution.pandas.udf.buffer.size": 4}): self.spark.range(10).repartition(1) \ .select(test_close(col("id"))).limit(2).collect() # wait here because python udf worker will take some time to detect diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 3ad3416256c7..08061886450d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1600,7 +1600,7 @@ object SQLConf { .createWithDefault(10000) val PANDAS_UDF_BUFFER_SIZE = - buildConf("spark.sql.pandas.udf.buffer.size") + buildConf("spark.sql.execution.pandas.udf.buffer.size") .doc( s"Same as ${BUFFER_SIZE} but only applies to Pandas UDF executions. If it is not set, " + s"the fallback is ${BUFFER_SIZE}. Note that Pandas execution requires more than 4 bytes. " +