From abebfe01e84d6a8c5499cf4017af473b7cb30651 Mon Sep 17 00:00:00 2001 From: Yuanjian Li Date: Thu, 14 Sep 2023 00:58:28 -0700 Subject: [PATCH] Add News, Release Note, Download Link for Apache Spark 3.5.0 - Vote Result: https://lists.apache.org/thread/kqqlsy90ormkccddtx6wgqmy8r1df8tx - Documentation: https://spark.apache.org/docs/3.5.0/ - Binary: https://downloads.apache.org/spark/spark-3.5.0/ - PySpark: pending on the File Limit Request https://github.com/pypi/support/issues/3175 - Maven Central: https://repo1.maven.org/maven2/org/apache/spark/spark-core_2.13/3.5.0/ image image image Author: Yuanjian Li Author: Li Yuanjian Closes #476 from xuanyuanking/3.5.0-release. --- Gemfile.lock | 2 +- documentation.md | 1 + downloads.md | 2 +- js/downloads.js | 1 + .../_posts/2023-09-13-spark-3-5-0-released.md | 14 + .../_posts/2023-09-13-spark-release-3-5-0.md | 336 ++++++++++ site/committers.html | 6 +- site/community.html | 6 +- site/contributing.html | 6 +- site/developer-tools.html | 6 +- site/docs/latest | 2 +- site/documentation.html | 7 +- site/downloads.html | 8 +- site/error-message-guidelines.html | 6 +- site/examples.html | 6 +- site/faq.html | 6 +- site/graphx/index.html | 6 +- site/history.html | 6 +- site/improvement-proposals.html | 6 +- site/js/downloads.js | 1 + site/mailing-lists.html | 6 +- site/mllib/index.html | 6 +- site/news/3-1-3-released.html | 6 +- site/news/amp-camp-2013-registration-ope.html | 6 +- .../announcing-the-first-spark-summit.html | 6 +- .../fourth-spark-screencast-published.html | 6 +- site/news/index.html | 15 +- site/news/new-repository-service.html | 6 +- .../next-official-release-spark-3.1.1.html | 6 +- site/news/nsdi-paper.html | 6 +- site/news/one-month-to-spark-summit-2015.html | 6 +- .../plan-for-dropping-python-2-support.html | 6 +- .../proposals-open-for-spark-summit-east.html | 6 +- ...gistration-open-for-spark-summit-east.html | 6 +- .../run-spark-and-shark-on-amazon-emr.html | 6 +- site/news/sigmod-system-award.html | 6 +- site/news/spark-0-6-1-and-0-5-2-released.html | 6 +- site/news/spark-0-6-2-released.html | 6 +- site/news/spark-0-7-0-released.html | 6 +- site/news/spark-0-7-2-released.html | 6 +- site/news/spark-0-7-3-released.html | 6 +- site/news/spark-0-8-0-released.html | 6 +- site/news/spark-0-8-1-released.html | 6 +- site/news/spark-0-9-0-released.html | 6 +- site/news/spark-0-9-1-released.html | 6 +- site/news/spark-0-9-2-released.html | 6 +- site/news/spark-1-0-0-released.html | 6 +- site/news/spark-1-0-1-released.html | 6 +- site/news/spark-1-0-2-released.html | 6 +- site/news/spark-1-1-0-released.html | 6 +- site/news/spark-1-1-1-released.html | 6 +- site/news/spark-1-2-0-released.html | 6 +- site/news/spark-1-2-1-released.html | 6 +- site/news/spark-1-2-2-released.html | 6 +- site/news/spark-1-3-0-released.html | 6 +- site/news/spark-1-4-0-released.html | 6 +- site/news/spark-1-4-1-released.html | 6 +- site/news/spark-1-5-0-released.html | 6 +- site/news/spark-1-5-1-released.html | 6 +- site/news/spark-1-5-2-released.html | 6 +- site/news/spark-1-6-0-released.html | 6 +- site/news/spark-1-6-1-released.html | 6 +- site/news/spark-1-6-2-released.html | 6 +- site/news/spark-1-6-3-released.html | 6 +- site/news/spark-2-0-0-released.html | 6 +- site/news/spark-2-0-1-released.html | 6 +- site/news/spark-2-0-2-released.html | 6 +- site/news/spark-2-1-0-released.html | 6 +- site/news/spark-2-1-1-released.html | 6 +- site/news/spark-2-1-2-released.html | 6 +- site/news/spark-2-1-3-released.html | 6 +- site/news/spark-2-2-0-released.html | 6 +- site/news/spark-2-2-1-released.html | 6 +- site/news/spark-2-2-2-released.html | 6 +- site/news/spark-2-3-0-released.html | 6 +- site/news/spark-2-3-1-released.html | 6 +- site/news/spark-2-3-2-released.html | 6 +- site/news/spark-2-3-3-released.html | 6 +- site/news/spark-2-3-4-released.html | 6 +- site/news/spark-2-4-0-released.html | 6 +- site/news/spark-2-4-1-released.html | 6 +- site/news/spark-2-4-2-released.html | 6 +- site/news/spark-2-4-3-released.html | 6 +- site/news/spark-2-4-4-released.html | 6 +- site/news/spark-2-4-5-released.html | 6 +- site/news/spark-2-4-6.html | 6 +- site/news/spark-2-4-7-released.html | 6 +- site/news/spark-2-4-8-released.html | 6 +- site/news/spark-2.0.0-preview.html | 6 +- site/news/spark-3-0-0-released.html | 6 +- site/news/spark-3-0-1-released.html | 6 +- site/news/spark-3-0-2-released.html | 6 +- site/news/spark-3-0-3-released.html | 6 +- site/news/spark-3-1-1-released.html | 6 +- site/news/spark-3-1-2-released.html | 6 +- site/news/spark-3-2-0-released.html | 6 +- site/news/spark-3-2-1-released.html | 6 +- site/news/spark-3-2-2-released.html | 6 +- site/news/spark-3-2-3-released.html | 6 +- site/news/spark-3-2-4-released.html | 6 +- site/news/spark-3-3-0-released.html | 6 +- site/news/spark-3-3-1-released.html | 6 +- site/news/spark-3-3-2-released.html | 6 +- site/news/spark-3-3-3-released.html | 6 +- site/news/spark-3-4-0-released.html | 6 +- site/news/spark-3-4-1-released.html | 6 +- site/news/spark-3-5-0-released.html | 200 ++++++ site/news/spark-3.0.0-preview.html | 6 +- site/news/spark-3.0.0-preview2.html | 6 +- .../spark-accepted-into-apache-incubator.html | 6 +- ...park-ai-summit-apr-2019-agenda-posted.html | 6 +- ...ark-ai-summit-june-2020-agenda-posted.html | 6 +- site/news/spark-and-shark-in-the-news.html | 6 +- site/news/spark-becomes-tlp.html | 6 +- site/news/spark-featured-in-wired.html | 6 +- .../spark-mailing-lists-moving-to-apache.html | 6 +- site/news/spark-meetups.html | 6 +- site/news/spark-release-2-2-3.html | 6 +- site/news/spark-screencasts-published.html | 6 +- site/news/spark-summit-2013-is-a-wrap.html | 6 +- .../news/spark-summit-2014-videos-posted.html | 6 +- .../news/spark-summit-2015-videos-posted.html | 6 +- site/news/spark-summit-agenda-posted.html | 6 +- .../spark-summit-east-2015-videos-posted.html | 6 +- .../spark-summit-east-2016-cfp-closing.html | 6 +- .../spark-summit-east-2017-agenda-posted.html | 6 +- .../spark-summit-east-agenda-posted-2015.html | 6 +- .../spark-summit-east-agenda-posted-2016.html | 6 +- .../spark-summit-eu-2017-agenda-posted.html | 6 +- .../spark-summit-europe-agenda-posted.html | 6 +- site/news/spark-summit-europe.html | 6 +- .../spark-summit-june-2016-agenda-posted.html | 6 +- .../spark-summit-june-2017-agenda-posted.html | 6 +- .../spark-summit-june-2018-agenda-posted.html | 6 +- .../spark-summit-oct-2018-agenda-posted.html | 6 +- site/news/spark-tips-from-quantifind.html | 6 +- ...spark-user-survey-and-powered-by-page.html | 6 +- site/news/spark-version-0-6-0-released.html | 6 +- .../spark-wins-cloudsort-100tb-benchmark.html | 6 +- ...ins-daytona-gray-sort-100tb-benchmark.html | 6 +- ...strata-exercises-now-available-online.html | 6 +- .../submit-talks-to-spark-summit-2014.html | 6 +- .../submit-talks-to-spark-summit-2016.html | 6 +- ...ubmit-talks-to-spark-summit-east-2016.html | 6 +- .../submit-talks-to-spark-summit-eu-2016.html | 6 +- site/news/two-weeks-to-spark-summit-2014.html | 6 +- ...o-from-first-spark-development-meetup.html | 6 +- site/powered-by.html | 6 +- site/release-process.html | 6 +- site/releases/spark-release-0-3.html | 6 +- site/releases/spark-release-0-5-0.html | 6 +- site/releases/spark-release-0-5-1.html | 6 +- site/releases/spark-release-0-5-2.html | 6 +- site/releases/spark-release-0-6-0.html | 6 +- site/releases/spark-release-0-6-1.html | 6 +- site/releases/spark-release-0-6-2.html | 6 +- site/releases/spark-release-0-7-0.html | 6 +- site/releases/spark-release-0-7-2.html | 6 +- site/releases/spark-release-0-7-3.html | 6 +- site/releases/spark-release-0-8-0.html | 6 +- site/releases/spark-release-0-8-1.html | 6 +- site/releases/spark-release-0-9-0.html | 6 +- site/releases/spark-release-0-9-1.html | 6 +- site/releases/spark-release-0-9-2.html | 6 +- site/releases/spark-release-1-0-0.html | 6 +- site/releases/spark-release-1-0-1.html | 6 +- site/releases/spark-release-1-0-2.html | 6 +- site/releases/spark-release-1-1-0.html | 6 +- site/releases/spark-release-1-1-1.html | 6 +- site/releases/spark-release-1-2-0.html | 6 +- site/releases/spark-release-1-2-1.html | 6 +- site/releases/spark-release-1-2-2.html | 6 +- site/releases/spark-release-1-3-0.html | 6 +- site/releases/spark-release-1-3-1.html | 6 +- site/releases/spark-release-1-4-0.html | 6 +- site/releases/spark-release-1-4-1.html | 6 +- site/releases/spark-release-1-5-0.html | 6 +- site/releases/spark-release-1-5-1.html | 6 +- site/releases/spark-release-1-5-2.html | 6 +- site/releases/spark-release-1-6-0.html | 6 +- site/releases/spark-release-1-6-1.html | 6 +- site/releases/spark-release-1-6-2.html | 6 +- site/releases/spark-release-1-6-3.html | 6 +- site/releases/spark-release-2-0-0.html | 6 +- site/releases/spark-release-2-0-1.html | 6 +- site/releases/spark-release-2-0-2.html | 6 +- site/releases/spark-release-2-1-0.html | 6 +- site/releases/spark-release-2-1-1.html | 6 +- site/releases/spark-release-2-1-2.html | 6 +- site/releases/spark-release-2-1-3.html | 6 +- site/releases/spark-release-2-2-0.html | 6 +- site/releases/spark-release-2-2-1.html | 6 +- site/releases/spark-release-2-2-2.html | 6 +- site/releases/spark-release-2-2-3.html | 6 +- site/releases/spark-release-2-3-0.html | 6 +- site/releases/spark-release-2-3-1.html | 6 +- site/releases/spark-release-2-3-2.html | 6 +- site/releases/spark-release-2-3-3.html | 6 +- site/releases/spark-release-2-3-4.html | 6 +- site/releases/spark-release-2-4-0.html | 6 +- site/releases/spark-release-2-4-1.html | 6 +- site/releases/spark-release-2-4-2.html | 6 +- site/releases/spark-release-2-4-3.html | 6 +- site/releases/spark-release-2-4-4.html | 6 +- site/releases/spark-release-2-4-5.html | 6 +- site/releases/spark-release-2-4-6.html | 6 +- site/releases/spark-release-2-4-7.html | 6 +- site/releases/spark-release-2-4-8.html | 6 +- site/releases/spark-release-3-0-0.html | 6 +- site/releases/spark-release-3-0-1.html | 6 +- site/releases/spark-release-3-0-2.html | 6 +- site/releases/spark-release-3-0-3.html | 6 +- site/releases/spark-release-3-1-1.html | 6 +- site/releases/spark-release-3-1-2.html | 6 +- site/releases/spark-release-3-1-3.html | 6 +- site/releases/spark-release-3-2-0.html | 6 +- site/releases/spark-release-3-2-1.html | 6 +- site/releases/spark-release-3-2-2.html | 6 +- site/releases/spark-release-3-2-3.html | 6 +- site/releases/spark-release-3-2-4.html | 6 +- site/releases/spark-release-3-3-0.html | 6 +- site/releases/spark-release-3-3-1.html | 6 +- site/releases/spark-release-3-3-2.html | 6 +- site/releases/spark-release-3-3-3.html | 6 +- site/releases/spark-release-3-4-0.html | 6 +- site/releases/spark-release-3-4-1.html | 6 +- site/releases/spark-release-3-5-0.html | 579 ++++++++++++++++++ site/research.html | 6 +- .../screencasts/1-first-steps-with-spark.html | 6 +- .../2-spark-documentation-overview.html | 6 +- .../3-transformations-and-caching.html | 6 +- .../4-a-standalone-job-in-spark.html | 6 +- site/screencasts/index.html | 6 +- site/security.html | 6 +- site/sitemap.xml | 16 +- site/sql/index.html | 6 +- site/streaming/index.html | 6 +- site/third-party-projects.html | 6 +- site/trademarks.html | 6 +- site/versioning-policy.html | 6 +- 240 files changed, 1845 insertions(+), 695 deletions(-) create mode 100644 news/_posts/2023-09-13-spark-3-5-0-released.md create mode 100644 releases/_posts/2023-09-13-spark-release-3-5-0.md create mode 100644 site/news/spark-3-5-0-released.html create mode 100644 site/releases/spark-release-3-5-0.html diff --git a/Gemfile.lock b/Gemfile.lock index a12f53745a4..f1e8cf7c6e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,7 +37,7 @@ GEM rexml kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) - liquid (4.0.3) + liquid (4.0.4) listen (3.4.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) diff --git a/documentation.md b/documentation.md index b2673f3128b..dfe33dbe6b1 100644 --- a/documentation.md +++ b/documentation.md @@ -12,6 +12,7 @@ navigation:

Setup instructions, programming guides, and other documentation are available for each stable version of Spark below:

    +
  • Spark 3.5.0
  • Spark 3.4.1
  • Spark 3.4.0
  • Spark 3.3.3
    • diff --git a/downloads.md b/downloads.md index 5952a1b5f93..3648d341380 100644 --- a/downloads.md +++ b/downloads.md @@ -35,7 +35,7 @@ Spark artifacts are [hosted in Maven Central](https://search.maven.org/search?q= groupId: org.apache.spark artifactId: spark-core_2.12 - version: 3.4.1 + version: 3.5.0 ### Installing with PyPi PySpark is now available in pypi. To install just run `pip install pyspark`. diff --git a/js/downloads.js b/js/downloads.js index 193b32f8cfd..5a76dbd9742 100644 --- a/js/downloads.js +++ b/js/downloads.js @@ -27,6 +27,7 @@ var packagesV13 = [hadoop3p, hadoop3pscala213, hadoop2p, hadoopFree, sources]; // 3.4.0+ var packagesV14 = [hadoop3p, hadoop3pscala213, hadoopFree, sources]; +addRelease("3.5.0", new Date("09/13/2023"), packagesV14, true); addRelease("3.4.1", new Date("06/23/2023"), packagesV14, true); addRelease("3.3.3", new Date("08/21/2023"), packagesV13, true); addRelease("3.2.4", new Date("04/13/2023"), packagesV12, true); diff --git a/news/_posts/2023-09-13-spark-3-5-0-released.md b/news/_posts/2023-09-13-spark-3-5-0-released.md new file mode 100644 index 00000000000..1f625b02004 --- /dev/null +++ b/news/_posts/2023-09-13-spark-3-5-0-released.md @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark 3.5.0 released +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +We are happy to announce the availability of Spark 3.5.0! Visit the release notes to read about the new features, or download the release today. diff --git a/releases/_posts/2023-09-13-spark-release-3-5-0.md b/releases/_posts/2023-09-13-spark-release-3-5-0.md new file mode 100644 index 00000000000..59671c6a18b --- /dev/null +++ b/releases/_posts/2023-09-13-spark-release-3-5-0.md @@ -0,0 +1,336 @@ +--- +layout: post +title: Spark Release 3.5.0 +categories: [] +tags: [] +status: publish +type: post +published: true +meta: +_edit_last: '4' +_wpas_done_all: '1' +--- + +Apache Spark 3.5.0 is the sixth release in the 3.x series. With significant contributions from the open-source community, this release addressed over 1,300 Jira tickets. + +This release introduces more scenarios with general availability for Spark Connect, like Scala and Go client, distributed training and inference support, and enhancement of compatibility for Structured streaming; introduces new PySpark and SQL functionality such as like SQL IDENTIFIER clause, named argument support for SQL function calls, SQL function support for HyperLogLog approximate aggregations, and Python user-defined table functions; simplifies distributed training with DeepSpeed; introduces watermark propagation among operators, introduces dropDuplicatesWithinWatermark operations in Structured Streaming. + +To download Apache Spark 3.5.0, please visit the [downloads](https://spark.apache.org/downloads.html) page. For [detailed changes](https://s.apache.org/spark-3.5.0), you can consult JIRA. We have also curated a list of high-level changes here, grouped by major modules. + + +* This will become a table of contents (this text will be scraped). +{:toc} + + +### Highlights + + + +* Scala and Go client support in Spark Connect [SPARK-42554](https://issues.apache.org/jira/browse/SPARK-42554) [SPARK-43351](https://issues.apache.org/jira/browse/SPARK-43351) +* PyTorch-based distributed ML Support for Spark Connect [SPARK-42471](https://issues.apache.org/jira/browse/SPARK-42471) +* Structured Streaming support for Spark Connect in Python and Scala [SPARK-42938](https://issues.apache.org/jira/browse/SPARK-42938) +* Pandas API support for the Python Spark Connect Client [SPARK-42497](https://issues.apache.org/jira/browse/SPARK-42497) +* Introduce Arrow Python UDFs [SPARK-40307](https://issues.apache.org/jira/browse/SPARK-40307) +* Support Python user-defined table functions [SPARK-43798](https://issues.apache.org/jira/browse/SPARK-43798) +* Migrate PySpark errors onto error classes [SPARK-42986](https://issues.apache.org/jira/browse/SPARK-42986) +* PySpark Test Framework [SPARK-44042](https://issues.apache.org/jira/browse/SPARK-44042) +* Add support for Datasketches HllSketch [SPARK-16484](https://issues.apache.org/jira/browse/SPARK-16484) +* Built-in SQL Function Improvement [SPARK-41231](https://issues.apache.org/jira/browse/SPARK-41231) +* IDENTIFIER clause [SPARK-43205](https://issues.apache.org/jira/browse/SPARK-43205) +* Add SQL functions into Scala, Python and R API [SPARK-43907](https://issues.apache.org/jira/browse/SPARK-43907) +* Add named argument support for SQL functions [SPARK-43922](https://issues.apache.org/jira/browse/SPARK-43922) +* Avoid unnecessary task rerun on decommissioned executor lost if shuffle data migrated [SPARK-41469](https://issues.apache.org/jira/browse/SPARK-41469) +* Distributed ML <> spark connect [SPARK-42471](https://issues.apache.org/jira/browse/SPARK-42471) +* DeepSpeed Distributor [SPARK-44264](https://issues.apache.org/jira/browse/SPARK-44264) +* Implement changelog checkpointing for RocksDB state store [SPARK-43421](https://issues.apache.org/jira/browse/SPARK-43421) +* Introduce watermark propagation among operators [SPARK-42376](https://issues.apache.org/jira/browse/SPARK-42376) +* Introduce dropDuplicatesWithinWatermark [SPARK-42931](https://issues.apache.org/jira/browse/SPARK-42931) +* RocksDB state store provider memory management enhancements [SPARK-43311](https://issues.apache.org/jira/browse/SPARK-43311) + + + +### Spark Connect + +* Refactoring of the sql module into sql and sql-api to produce a minimum set of dependencies that can be shared between the Scala Spark Connect client and Spark and avoids pulling all of the Spark transitive dependencies. [SPARK-44273](https://issues.apache.org/jira/browse/SPARK-44273) +* Introducing the Scala client for Spark Connect [SPARK-42554](https://issues.apache.org/jira/browse/SPARK-42554) +* Pandas API support for the Python Spark Connect Client [SPARK-42497](https://issues.apache.org/jira/browse/SPARK-42497) +* PyTorch-based distributed ML Support for Spark Connect [SPARK-42471](https://issues.apache.org/jira/browse/SPARK-42471) +* Structured Streaming support for Spark Connect in Python and Scala [SPARK-42938](https://issues.apache.org/jira/browse/SPARK-42938) +* Initial version of the Go client [SPARK-43351](https://issues.apache.org/jira/browse/SPARK-43351) +* Lot’s of compatibility improvements between Spark native and the Spark Connect clients across Python and Scala +* Improved debugability and request handling for client applications (asynchronous processing, retries, long-lived queries) + + +### Spark SQL + +#### Features + + +* Add metadata column file block start and length [SPARK-42423](https://issues.apache.org/jira/browse/SPARK-42423) +* Support positional parameters in Scala/Java sql() [SPARK-44066](https://issues.apache.org/jira/browse/SPARK-44066) +* Add named parameter support in parser for function calls [SPARK-43922](https://issues.apache.org/jira/browse/SPARK-43922) +* Support SELECT DEFAULT with ORDER BY, LIMIT, OFFSET for INSERT source relation [SPARK-43071](https://issues.apache.org/jira/browse/SPARK-43071) +* Add SQL grammar for PARTITION BY and ORDER BY clause after TABLE arguments for TVF calls [SPARK-44503](https://issues.apache.org/jira/browse/SPARK-44503) +* Include column default values in DESCRIBE and SHOW CREATE TABLE output [SPARK-42123](https://issues.apache.org/jira/browse/SPARK-42123) +* Add optional pattern for Catalog.listCatalogs [SPARK-43792](https://issues.apache.org/jira/browse/SPARK-43792) +* Add optional pattern for Catalog.listDatabases [SPARK-43881](https://issues.apache.org/jira/browse/SPARK-43881) +* Callback when ready for execution [SPARK-44145](https://issues.apache.org/jira/browse/SPARK-44145) +* Support Insert By Name statement [SPARK-42750](https://issues.apache.org/jira/browse/SPARK-42750) +* Add call_function for Scala API [SPARK-44131](https://issues.apache.org/jira/browse/SPARK-44131) +* Stable derived column aliases [SPARK-40822](https://issues.apache.org/jira/browse/SPARK-40822) +* Support general constant expressions as CREATE/REPLACE TABLE OPTIONS values [SPARK-43529](https://issues.apache.org/jira/browse/SPARK-43529) +* Support subqueries with correlation through INTERSECT/EXCEPT [SPARK-36124](https://issues.apache.org/jira/browse/SPARK-36124) +* IDENTIFIER clause [SPARK-43205](https://issues.apache.org/jira/browse/SPARK-43205) +* ANSI MODE: Conv should return an error if the internal conversion overflows [SPARK-42427](https://issues.apache.org/jira/browse/SPARK-42427) + +#### Functions + +* Add support for Datasketches HllSketch [SPARK-16484](https://issues.apache.org/jira/browse/SPARK-16484) +* Support the CBC mode by aes_encrypt()/aes_decrypt() [SPARK-43038](https://issues.apache.org/jira/browse/SPARK-43038) +* Support TABLE argument parser rule for TableValuedFunction [SPARK-44200](https://issues.apache.org/jira/browse/SPARK-44200) +* Implement bitmap functions [SPARK-44154](https://issues.apache.org/jira/browse/SPARK-44154) +* Add the try_aes_decrypt() function [SPARK-42701](https://issues.apache.org/jira/browse/SPARK-42701) +* array_insert should fail with 0 index [SPARK-43011](https://issues.apache.org/jira/browse/SPARK-43011) +* Add to_varchar alias for to_char [SPARK-43815](https://issues.apache.org/jira/browse/SPARK-43815) +* High-order function: array_compact implementation [SPARK-41235](https://issues.apache.org/jira/browse/SPARK-41235) +* Add analyzer support of named arguments for built-in functions [SPARK-44059](https://issues.apache.org/jira/browse/SPARK-44059) +* Add NULLs for INSERTs with user-specified lists of fewer columns than the target table [SPARK-42521](https://issues.apache.org/jira/browse/SPARK-42521) +* Adds support for aes_encrypt IVs and AAD [SPARK-43290](https://issues.apache.org/jira/browse/SPARK-43290) +* DECODE function returns wrong results when passed NULL [SPARK-41668](https://issues.apache.org/jira/browse/SPARK-41668) +* Support udf 'luhn_check' [SPARK-42191](https://issues.apache.org/jira/browse/SPARK-42191) +* Support implicit lateral column alias resolution on Aggregate [SPARK-41631](https://issues.apache.org/jira/browse/SPARK-41631) +* Support implicit lateral column alias in queries with Window [SPARK-42217](https://issues.apache.org/jira/browse/SPARK-42217) +* Add 3-args function aliases DATE_ADD and DATE_DIFF [SPARK-43492](https://issues.apache.org/jira/browse/SPARK-43492) + +#### Data Sources + +* Char/Varchar Support for JDBC Catalog [SPARK-42904](https://issues.apache.org/jira/browse/SPARK-42904) +* Support Get SQL Keywords Dynamically Thru JDBC API and TVF [SPARK-43119](https://issues.apache.org/jira/browse/SPARK-43119) +* DataSource V2: Handle MERGE commands for delta-based sources [SPARK-43885](https://issues.apache.org/jira/browse/SPARK-43885) +* DataSource V2: Handle MERGE commands for group-based sources [SPARK-43963](https://issues.apache.org/jira/browse/SPARK-43963) +* DataSource V2: Handle UPDATE commands for group-based sources [SPARK-43975](https://issues.apache.org/jira/browse/SPARK-43975) +* DataSource V2: Allow representing updates as deletes and inserts [SPARK-43775](https://issues.apache.org/jira/browse/SPARK-43775) +* Allow jdbc dialects to override the query used to create a table [SPARK-41516](https://issues.apache.org/jira/browse/SPARK-41516) +* SPJ: Support partially clustered distribution [SPARK-42038](https://issues.apache.org/jira/browse/SPARK-42038) +* DSv2 allows CTAS/RTAS to reserve schema nullability [SPARK-43390](https://issues.apache.org/jira/browse/SPARK-43390) +* Add spark.sql.files.maxPartitionNum [SPARK-44021](https://issues.apache.org/jira/browse/SPARK-44021) +* Handle UPDATE commands for delta-based sources [SPARK-43324](https://issues.apache.org/jira/browse/SPARK-43324) +* Allow V2 writes to indicate advisory shuffle partition size [SPARK-42779](https://issues.apache.org/jira/browse/SPARK-42779) +* Support lz4raw compression codec for Parquet [SPARK-43273](https://issues.apache.org/jira/browse/SPARK-43273) +* Avro: writing complex unions [SPARK-25050](https://issues.apache.org/jira/browse/SPARK-25050) +* Speed up Timestamp type inference with user-provided format in JSON/CSV data source [SPARK-39280](https://issues.apache.org/jira/browse/SPARK-39280) +* Avro to Support custom decimal type backed by Long [SPARK-43901](https://issues.apache.org/jira/browse/SPARK-43901) +* Avoid shuffle in Storage-Partitioned Join when partition keys mismatch, but join expressions are compatible [SPARK-41413](https://issues.apache.org/jira/browse/SPARK-41413) +* Change binary to unsupported dataType in CSV format [SPARK-42237](https://issues.apache.org/jira/browse/SPARK-42237) +* Allow Avro to convert union type to SQL with field name stable with type [SPARK-43333](https://issues.apache.org/jira/browse/SPARK-43333) +* Speed up Timestamp type inference with legacy format in JSON/CSV data source [SPARK-39281](https://issues.apache.org/jira/browse/SPARK-39281) + +#### Query Optimization + +* Subexpression elimination support shortcut expression [SPARK-42815](https://issues.apache.org/jira/browse/SPARK-42815) +* Improve join stats estimation if one side can keep uniqueness [SPARK-39851](https://issues.apache.org/jira/browse/SPARK-39851) +* Introduce the group limit of Window for rank-based filter to optimize top-k computation [SPARK-37099](https://issues.apache.org/jira/browse/SPARK-37099) +* Fix behavior of null IN (empty list) in optimization rules [SPARK-44431](https://issues.apache.org/jira/browse/SPARK-44431) +* Infer and push down window limit through window if partitionSpec is empty [SPARK-41171](https://issues.apache.org/jira/browse/SPARK-41171) +* Remove the outer join if they are all distinct aggregate functions [SPARK-42583](https://issues.apache.org/jira/browse/SPARK-42583) +* Collapse two adjacent windows with the same partition/order in subquery [SPARK-42525](https://issues.apache.org/jira/browse/SPARK-42525) +* Push down limit through Python UDFs [SPARK-42115](https://issues.apache.org/jira/browse/SPARK-42115) +* Optimize the order of filtering predicates [SPARK-40045](https://issues.apache.org/jira/browse/SPARK-40045) + +#### Code Generation and Query Execution + +* Runtime filter should supports multi level shuffle join side as filter creation side [SPARK-41674](https://issues.apache.org/jira/browse/SPARK-41674) +* Codegen Support for HiveSimpleUDF [SPARK-42052](https://issues.apache.org/jira/browse/SPARK-42052) +* Codegen Support for HiveGenericUDF [SPARK-42051](https://issues.apache.org/jira/browse/SPARK-42051) +* Codegen Support for build side outer shuffled hash join [SPARK-44060](https://issues.apache.org/jira/browse/SPARK-44060) +* Implement code generation for to_csv function (StructsToCsv) [SPARK-42169](https://issues.apache.org/jira/browse/SPARK-42169) +* Make AQE support InMemoryTableScanExec [SPARK-42101](https://issues.apache.org/jira/browse/SPARK-42101) +* Support left outer join build left or right outer join build right in shuffled hash join [SPARK-36612](https://issues.apache.org/jira/browse/SPARK-36612) +* Respect RequiresDistributionAndOrdering in CTAS/RTAS [SPARK-43088](https://issues.apache.org/jira/browse/SPARK-43088) +* Coalesce buckets in join applied on broadcast join stream side [SPARK-43107](https://issues.apache.org/jira/browse/SPARK-43107) +* Set nullable correctly on coalesced join key in full outer USING join [SPARK-44251](https://issues.apache.org/jira/browse/SPARK-44251) +* Fix IN subquery ListQuery nullability [SPARK-43413](https://issues.apache.org/jira/browse/SPARK-43413) + +#### Other Notable Changes + +* Set nullable correctly for keys in USING joins [SPARK-43718](https://issues.apache.org/jira/browse/SPARK-43718) +* Fix COUNT(*) is null bug in correlated scalar subquery [SPARK-43156](https://issues.apache.org/jira/browse/SPARK-43156) +* Dataframe.joinWith outer-join should return a null value for unmatched row [SPARK-37829](https://issues.apache.org/jira/browse/SPARK-37829) +* Automatically rename conflicting metadata columns [SPARK-42683](https://issues.apache.org/jira/browse/SPARK-42683) +* Document the Spark SQL error classes in user-facing documentation [SPARK-42706](https://issues.apache.org/jira/browse/SPARK-42706) + +### PySpark + + +#### Features + + +* Support positional parameters in Python sql() [SPARK-44140](https://issues.apache.org/jira/browse/SPARK-44140) +* Support parameterized SQL by sql() [SPARK-41666](https://issues.apache.org/jira/browse/SPARK-41666) +* Support Python user-defined table functions [SPARK-43797](https://issues.apache.org/jira/browse/SPARK-43797) +* Support to set Python executable for UDF and pandas function APIs in workers during runtime [SPARK-43574](https://issues.apache.org/jira/browse/SPARK-43574) +* Add DataFrame.offset to PySpark [SPARK-43213](https://issues.apache.org/jira/browse/SPARK-43213) +* Implement __dir__() in pyspark.sql.dataframe.DataFrame to include columns [SPARK-43270](https://issues.apache.org/jira/browse/SPARK-43270) +* Add option to use large variable width vectors for arrow UDF operations [SPARK-39979](https://issues.apache.org/jira/browse/SPARK-39979) +* Make mapInPandas / mapInArrow support barrier mode execution [SPARK-42896](https://issues.apache.org/jira/browse/SPARK-42896) +* Add JobTag APIs to PySpark SparkContext [SPARK-44194](https://issues.apache.org/jira/browse/SPARK-44194) +* Support for Python UDTF to analyze in Python [SPARK-44380](https://issues.apache.org/jira/browse/SPARK-44380) +* Expose TimestampNTZType in pyspark.sql.types [SPARK-43759](https://issues.apache.org/jira/browse/SPARK-43759) +* Support nested timestamp type [SPARK-43545](https://issues.apache.org/jira/browse/SPARK-43545) +* Support UserDefinedType in createDataFrame from pandas DataFrame and toPandas [[SPARK-43817](https://issues.apache.org/jira/browse/SPARK-43817)][SPARK-43702](https://issues.apache.org/jira/browse/SPARK-43702) +* Add descriptor binary option to Pyspark Protobuf API [SPARK-43799](https://issues.apache.org/jira/browse/SPARK-43799) +* Accept generics tuple as typing hints of Pandas UDF [SPARK-43886](https://issues.apache.org/jira/browse/SPARK-43886) +* Add array_prepend function [SPARK-41233](https://issues.apache.org/jira/browse/SPARK-41233) +* Add assertDataFrameEqual util function [SPARK-44061](https://issues.apache.org/jira/browse/SPARK-44061) +* Support arrow-optimized Python UDTFs [SPARK-43964](https://issues.apache.org/jira/browse/SPARK-43964) +* Allow custom precision for fp approx equality [SPARK-44217](https://issues.apache.org/jira/browse/SPARK-44217) +* Make assertSchemaEqual API public [SPARK-44216](https://issues.apache.org/jira/browse/SPARK-44216) +* Support fill_value for ps.Series [SPARK-42094](https://issues.apache.org/jira/browse/SPARK-42094) +* Support struct type in createDataFrame from pandas DataFrame [SPARK-43473](https://issues.apache.org/jira/browse/SPARK-43473) + +#### Other Notable Changes + +* Add autocomplete support for df[|] in pyspark.sql.dataframe.DataFrame [[SPARK-43892](https://issues.apache.org/jira/browse/SPARK-43892)] +* Deprecate & remove the APIs that will be removed in pandas 2.0 [[SPARK-42593](https://issues.apache.org/jira/browse/SPARK-42593)] +* Make Python the first tab for code examples - Spark SQL, DataFrames and Datasets Guide [SPARK-42493](https://issues.apache.org/jira/browse/SPARK-42493) +* Updating remaining Spark documentation code examples to show Python by default [SPARK-42642](https://issues.apache.org/jira/browse/SPARK-42642) +* Use deduplicated field names when creating Arrow RecordBatch [[SPARK-41971](https://issues.apache.org/jira/browse/SPARK-41971)] +* Support duplicated field names in createDataFrame with pandas DataFrame [[SPARK-43528](https://issues.apache.org/jira/browse/SPARK-43528)] +* Allow columns parameter when creating DataFrame with Series [[SPARK-42194](https://issues.apache.org/jira/browse/SPARK-42194)] + +### Core + +* Schedule mergeFinalize when push merge shuffleMapStage retry but no running tasks [SPARK-40082](https://issues.apache.org/jira/browse/SPARK-40082) +* Introduce PartitionEvaluator for SQL operator execution [SPARK-43061](https://issues.apache.org/jira/browse/SPARK-43061) +* Allow ShuffleDriverComponent to declare if shuffle data is reliably stored [SPARK-42689](https://issues.apache.org/jira/browse/SPARK-42689) +* Add max attempts limitation for stages to avoid potential infinite retry [SPARK-42577](https://issues.apache.org/jira/browse/SPARK-42577) +* Support log level configuration with static Spark conf [SPARK-43782](https://issues.apache.org/jira/browse/SPARK-43782) +* Optimize PercentileHeap [SPARK-42528](https://issues.apache.org/jira/browse/SPARK-42528) +* Add reason argument to TaskScheduler.cancelTasks [SPARK-42602](https://issues.apache.org/jira/browse/SPARK-42602) +* Avoid unnecessary task rerun on decommissioned executor lost if shuffle data migrated [SPARK-41469](https://issues.apache.org/jira/browse/SPARK-41469) +* Fixing accumulator undercount in the case of the retry task with rdd cache [SPARK-41497](https://issues.apache.org/jira/browse/SPARK-41497) +* Use RocksDB for spark.history.store.hybridStore.diskBackend by default [SPARK-42277](https://issues.apache.org/jira/browse/SPARK-42277) +* Support spark.kubernetes.setSubmitTimeInDriver [SPARK-43014](https://issues.apache.org/jira/browse/SPARK-43014) +* NonFateSharingCache wrapper for Guava Cache [SPARK-43300](https://issues.apache.org/jira/browse/SPARK-43300) +* Improve the performance of MapOutputTracker.updateMapOutput [SPARK-43043](https://issues.apache.org/jira/browse/SPARK-43043) +* Allowing apps to control whether their metadata gets saved in the db by the External Shuffle Service [SPARK-43179](https://issues.apache.org/jira/browse/SPARK-43179) +* Port executor failure tracker from Spark on YARN to K8s [SPARK-41210](https://issues.apache.org/jira/browse/SPARK-41210) +* Parameterize the max number of attempts for driver props fetcher in KubernetesExecutorBackend [SPARK-42764](https://issues.apache.org/jira/browse/SPARK-42764) +* Add SPARK_DRIVER_POD_IP env variable to executor pods [SPARK-42769](https://issues.apache.org/jira/browse/SPARK-42769) +* Mounts the hadoop config map on the executor pod [SPARK-43504](https://issues.apache.org/jira/browse/SPARK-43504) + + +### Structured Streaming + +* Add support for tracking pinned blocks memory usage for RocksDB state store [SPARK-43120](https://issues.apache.org/jira/browse/SPARK-43120) +* Add RocksDB state store provider memory management enhancements [SPARK-43311](https://issues.apache.org/jira/browse/SPARK-43311) +* Introduce dropDuplicatesWithinWatermark [SPARK-42931](https://issues.apache.org/jira/browse/SPARK-42931) +* Introduce a new callback onQueryIdle() to StreamingQueryListener [SPARK-43183](https://issues.apache.org/jira/browse/SPARK-43183) +* Add option to skip commit coordinator as part of StreamingWrite API for DSv2 sources/sinks [SPARK-42968](https://issues.apache.org/jira/browse/SPARK-42968) +* Introduce a new callback "onQueryIdle" to StreamingQueryListener [SPARK-43183](https://issues.apache.org/jira/browse/SPARK-43183) +* Implement Changelog based Checkpointing for RocksDB State Store Provider [SPARK-43421](https://issues.apache.org/jira/browse/SPARK-43421) +* Add support for WRITE_FLUSH_BYTES for RocksDB used in streaming stateful operators [SPARK-42792](https://issues.apache.org/jira/browse/SPARK-42792) +* Add support for setting max_write_buffer_number and write_buffer_size for RocksDB used in streaming [SPARK-42819](https://issues.apache.org/jira/browse/SPARK-42819) +* RocksDB StateStore lock acquisition should happen after getting input iterator from inputRDD [SPARK-42566](https://issues.apache.org/jira/browse/SPARK-42566) +* Introduce watermark propagation among operators [SPARK-42376](https://issues.apache.org/jira/browse/SPARK-42376) +* Cleanup orphan sst and log files in RocksDB checkpoint directory [SPARK-42353](https://issues.apache.org/jira/browse/SPARK-42353) +* Expand QueryTerminatedEvent to contain error class if it exists in exception [SPARK-43482](https://issues.apache.org/jira/browse/SPARK-43482) + +### ML + +* Support Distributed Training of Functions Using Deepspeed [SPARK-44264](https://issues.apache.org/jira/browse/SPARK-44264) +* Base interfaces of sparkML for spark3.5: estimator/transformer/model/evaluator [SPARK-43516](https://issues.apache.org/jira/browse/SPARK-43516) +* Make MLv2 (ML on spark connect) supports pandas >= 2.0 [SPARK-43783](https://issues.apache.org/jira/browse/SPARK-43783) +* Update MLv2 Transformer interfaces [SPARK-43516](https://issues.apache.org/jira/browse/SPARK-43516) +* New pyspark ML logistic regression estimator implemented on top of distributor [SPARK-43097](https://issues.apache.org/jira/browse/SPARK-43097) +* Add Classifier.getNumClasses back [SPARK-42526](https://issues.apache.org/jira/browse/SPARK-42526) +* Write a Deepspeed Distributed Learning Class DeepspeedTorchDistributor [SPARK-44264](https://issues.apache.org/jira/browse/SPARK-44264) +* Basic saving / loading implementation for ML on spark connect [SPARK-43981](https://issues.apache.org/jira/browse/SPARK-43981) +* Improve logistic regression model saving [SPARK-43097](https://issues.apache.org/jira/browse/SPARK-43097) +* Implement pipeline estimator for ML on spark connect [SPARK-43982](https://issues.apache.org/jira/browse/SPARK-43982) +* Implement cross validator estimator [SPARK-43983](https://issues.apache.org/jira/browse/SPARK-43983) +* Implement classification evaluator [SPARK-44250](https://issues.apache.org/jira/browse/SPARK-44250) +* Make PyTorch Distributor compatible with Spark Connect [SPARK-42993](https://issues.apache.org/jira/browse/SPARK-42993) + +### UI + +* Add a Spark UI page for Spark Connect [SPARK-44394](https://issues.apache.org/jira/browse/SPARK-44394) +* Support Heap Histogram column in Executors tab [SPARK-44153](https://issues.apache.org/jira/browse/SPARK-44153) +* Show error message on UI for each failed query [SPARK-44367](https://issues.apache.org/jira/browse/SPARK-44367) +* Display Add/Remove Time of Executors on Executors Tab [SPARK-44309](https://issues.apache.org/jira/browse/SPARK-44309) + +### Build and Others + +* Remove Python 3.7 Support [SPARK-43347](https://issues.apache.org/jira/browse/SPARK-43347) +* Increate PyArrow minimum version to 4.0.0 [SPARK-44183](https://issues.apache.org/jira/browse/SPARK-44183) +* Support R 4.3.1 [SPARK-43447](https://issues.apache.org/jira/browse/SPARK-43447) [SPARK-44192](https://issues.apache.org/jira/browse/SPARK-44192) +* Add JobTag APIs to SparkR SparkContext [SPARK-44195](https://issues.apache.org/jira/browse/SPARK-44195) +* Add math functions to SparkR [SPARK-44349](https://issues.apache.org/jira/browse/SPARK-44349) +* Upgrade Parquet to 1.13.1 [SPARK-43519](https://issues.apache.org/jira/browse/SPARK-43519) +* Upgrade kubernetes-client to 6.7.2 [SPARK-42362](https://issues.apache.org/jira/browse/SPARK-42362) [SPARK-42761](https://issues.apache.org/jira/browse/SPARK-42761) [SPARK-42885](https://issues.apache.org/jira/browse/SPARK-42885) [SPARK-43355](https://issues.apache.org/jira/browse/SPARK-43355) [SPARK-43581](https://issues.apache.org/jira/browse/SPARK-43581) [SPARK-43950](https://issues.apache.org/jira/browse/SPARK-43950) [SPARK-43990](https://issues.apache.org/jira/browse/SPARK-43990) +* Upgrade ASM to 9.5 [SPARK-43537](https://issues.apache.org/jira/browse/SPARK-43537) [SPARK-43588](https://issues.apache.org/jira/browse/SPARK-43588) +* Upgrade rocksdbjni to 8.3.2 [SPARK-41569](https://issues.apache.org/jira/browse/SPARK-41569) [SPARK-42718](https://issues.apache.org/jira/browse/SPARK-42718) [SPARK-43007](https://issues.apache.org/jira/browse/SPARK-43007) [SPARK-43436](https://issues.apache.org/jira/browse/SPARK-43436) [SPARK-44256](https://issues.apache.org/jira/browse/SPARK-44256) +* Upgrade Netty to 4.1.93 [SPARK-42218](https://issues.apache.org/jira/browse/SPARK-42218) [SPARK-42417](https://issues.apache.org/jira/browse/SPARK-42417) [SPARK-42487](https://issues.apache.org/jira/browse/SPARK-42487) [SPARK-43609](https://issues.apache.org/jira/browse/SPARK-43609) [SPARK-44128](https://issues.apache.org/jira/browse/SPARK-44128) +* Upgrade zstd-jni to 1.5.5-5 [SPARK-42409](https://issues.apache.org/jira/browse/SPARK-42409) [SPARK-42625](https://issues.apache.org/jira/browse/SPARK-42625) [SPARK-43080](https://issues.apache.org/jira/browse/SPARK-43080) [SPARK-43294](https://issues.apache.org/jira/browse/SPARK-43294) [SPARK-43737](https://issues.apache.org/jira/browse/SPARK-43737) [SPARK-43994](https://issues.apache.org/jira/browse/SPARK-43994) [SPARK-44465](https://issues.apache.org/jira/browse/SPARK-44465) +* Upgrade dropwizard metrics 4.2.19 [SPARK-42654](https://issues.apache.org/jira/browse/SPARK-42654) [SPARK-43738](https://issues.apache.org/jira/browse/SPARK-43738) [SPARK-44296](https://issues.apache.org/jira/browse/SPARK-44296) +* Upgrade gcs-connector to 2.2.14 [SPARK-42888](https://issues.apache.org/jira/browse/SPARK-42888) [SPARK-43842](https://issues.apache.org/jira/browse/SPARK-43842) +* Upgrade commons-crypto to 1.2.0 [SPARK-42488](https://issues.apache.org/jira/browse/SPARK-42488) +* Upgrade scala-parser-combinators from 2.1.1 to 2.2.0 [SPARK-42489](https://issues.apache.org/jira/browse/SPARK-42489) +* Upgrade protobuf-java to 3.23.4 [SPARK-41711](https://issues.apache.org/jira/browse/SPARK-41711) [SPARK-42490](https://issues.apache.org/jira/browse/SPARK-42490) [SPARK-42798](https://issues.apache.org/jira/browse/SPARK-42798) [SPARK-43899](https://issues.apache.org/jira/browse/SPARK-43899) [SPARK-44382](https://issues.apache.org/jira/browse/SPARK-44382) +* Upgrade commons-codec to 1.16.0 [SPARK-44151](https://issues.apache.org/jira/browse/SPARK-44151) +* Upgrade Apache Kafka to 3.4.1 [SPARK-42396](https://issues.apache.org/jira/browse/SPARK-42396) [SPARK-44181](https://issues.apache.org/jira/browse/SPARK-44181) +* Upgrade RoaringBitmap to 0.9.45 [SPARK-42385](https://issues.apache.org/jira/browse/SPARK-42385) [SPARK-43495](https://issues.apache.org/jira/browse/SPARK-43495) [SPARK-44221](https://issues.apache.org/jira/browse/SPARK-44221) +* Update ORC to 1.9.0 [SPARK-42820](https://issues.apache.org/jira/browse/SPARK-42820) [SPARK-44053](https://issues.apache.org/jira/browse/SPARK-44053) [SPARK-44231](https://issues.apache.org/jira/browse/SPARK-44231) +* Upgrade to Avro 1.11.2 [SPARK-44277](https://issues.apache.org/jira/browse/SPARK-44277) +* Upgrade commons-compress to 1.23.0 [SPARK-43102](https://issues.apache.org/jira/browse/SPARK-43102) +* Upgrade joda-time from 2.12.2 to 2.12.5 [SPARK-43008](https://issues.apache.org/jira/browse/SPARK-43008) +* Upgrade snappy-java to 1.1.10.3 [SPARK-42242](https://issues.apache.org/jira/browse/SPARK-42242) [SPARK-43758](https://issues.apache.org/jira/browse/SPARK-43758) [SPARK-44070](https://issues.apache.org/jira/browse/SPARK-44070) [SPARK-44415](https://issues.apache.org/jira/browse/SPARK-44415) [SPARK-44513](https://issues.apache.org/jira/browse/SPARK-44513) +* Upgrade mysql-connector-java from 8.0.31 to 8.0.32 [SPARK-42717](https://issues.apache.org/jira/browse/SPARK-42717) +* Upgrade Apache Arrow to 12.0.1 [SPARK-42161](https://issues.apache.org/jira/browse/SPARK-42161) [SPARK-43446](https://issues.apache.org/jira/browse/SPARK-43446) [SPARK-44094](https://issues.apache.org/jira/browse/SPARK-44094) +* Upgrade commons-io to 2.12.0 [SPARK-43739](https://issues.apache.org/jira/browse/SPARK-43739) +* Upgrade Apache commons-io to 2.13.0 [SPARK-43739](https://issues.apache.org/jira/browse/SPARK-43739) [SPARK-44028](https://issues.apache.org/jira/browse/SPARK-44028) +* Upgrade FasterXML jackson to 2.15.2 [SPARK-42354](https://issues.apache.org/jira/browse/SPARK-42354) [SPARK-43774](https://issues.apache.org/jira/browse/SPARK-43774) [SPARK-43904](https://issues.apache.org/jira/browse/SPARK-43904) +* Upgrade log4j2 to 2.20.0 [SPARK-42536](https://issues.apache.org/jira/browse/SPARK-42536) +* Upgrade slf4j to 2.0.7 [SPARK-42871](https://issues.apache.org/jira/browse/SPARK-42871) +* Upgrade numpy and pandas in the release Dockerfile [SPARK-42524](https://issues.apache.org/jira/browse/SPARK-42524) +* Upgrade Jersey to 2.40 [SPARK-44316](https://issues.apache.org/jira/browse/SPARK-44316) +* Upgrade H2 from 2.1.214 to 2.2.220 [SPARK-44393](https://issues.apache.org/jira/browse/SPARK-44393) +* Upgrade optionator to ^0.9.3 [SPARK-44279](https://issues.apache.org/jira/browse/SPARK-44279) +* Upgrade bcprov-jdk15on and bcpkix-jdk15on to 1.70 [SPARK-44441](https://issues.apache.org/jira/browse/SPARK-44441) +* Upgrade mlflow to 2.3.1 [SPARK-43344](https://issues.apache.org/jira/browse/SPARK-43344) +* Upgrade Tink to 1.9.0 [SPARK-42780](https://issues.apache.org/jira/browse/SPARK-42780) +* Upgrade silencer to 1.7.13 [SPARK-41787](https://issues.apache.org/jira/browse/SPARK-41787) [SPARK-44031](https://issues.apache.org/jira/browse/SPARK-44031) +* Upgrade Ammonite to 2.5.9 [SPARK-44041](https://issues.apache.org/jira/browse/SPARK-44041) +* Upgrade Scala to 2.12.18 [SPARK-43832](https://issues.apache.org/jira/browse/SPARK-43832) +* Upgrade org.scalatestplus:selenium-4-4 to org.scalatestplus:selenium-4-7 [SPARK-41587](https://issues.apache.org/jira/browse/SPARK-41587) +* Upgrade minimatch to 3.1.2 [SPARK-41634](https://issues.apache.org/jira/browse/SPARK-41634) +* Upgrade sbt-assembly from 2.0.0 to 2.1.0 [SPARK-41704](https://issues.apache.org/jira/browse/SPARK-41704) +* Update maven-checkstyle-plugin from 3.1.2 to 3.2.0 [SPARK-41714](https://issues.apache.org/jira/browse/SPARK-41714) +* Upgrade dev.ludovic.netlib to 3.0.3 [SPARK-41750](https://issues.apache.org/jira/browse/SPARK-41750) +* Upgrade hive-storage-api to 2.8.1 [SPARK-41798](https://issues.apache.org/jira/browse/SPARK-41798) +* Upgrade Apache httpcore to 4.4.16 [SPARK-41802](https://issues.apache.org/jira/browse/SPARK-41802) +* Upgrade jetty to 9.4.52.v20230823 [SPARK-45052](https://issues.apache.org/jira/browse/SPARK-45052) +* Upgrade compress-lzf to 1.1.2 [SPARK-42274](https://issues.apache.org/jira/browse/SPARK-42274) + +### Removals, Behavior Changes and Deprecations + +#### Upcoming Removal +The following features will be removed in the next Spark major release +- Support for Java 8 and Java 11, and the minimal supported Java version will be Java 17 +- Support for Scala 2.12, and the minimal supported Scala version will be 2.13 + +#### Migration Guides +- [Spark Core](https://spark.apache.org/docs/3.5.0/core-migration-guide.html) +- [SQL, Datasets, and DataFrame](https://spark.apache.org/docs/3.5.0/sql-migration-guide.html) +- [Structured Streaming](https://spark.apache.org/docs/3.5.0/ss-migration-guide.html) +- [MLlib (Machine Learning)](https://spark.apache.org/docs/3.5.0/ml-migration-guide.html) +- [PySpark (Python on Spark)](https://spark.apache.org/docs/3.5.0/api/python/migration_guide/pyspark_upgrade.html) +- [SparkR (R on Spark)](https://spark.apache.org/docs/3.5.0/sparkr-migration-guide.html) + + +### Credits + +Last but not least, this release would not have been possible without the following contributors: +Adam Binford, Ahmed Hussein, Alex Jing, Alice Sayutina, Alkis Evlogimenos, Allan Folting, Allison Portis, Allison Wang, Amanda Liu, Andrey Gubichev, Andy Grove, Anish Shrigondekar, Anton Okolnychyi, Bartosz Konieczny, Beishao Cao, Bjørn Jørgensen, Bo Zhang, Bruce Robbins, Chandni Singh, Chao Sun, Chaoqin Li, Cheng Pan, Christopher Cooper, Christopher Watford, ConeyLiu, Daniel Sparing, Daniel Tenedorio, David Lewis, Dongjoon Hyun, Emil Ejbyfeldt, Enrico Minack, Eric Marnadi, Eric Ogren, Erik Krogen, Eugene Gusev, Fei Wang, Fokko Driesprong, Frank Yin, Fu Chen, Gene Pang, Gengliang Wang, Gera Shegalov, Giambattista Bloisi, Guilhem Vuillier, Gurpreet Singh, Haejoon Lee, Harsh Motwani, Henry Mai, Herman Van Hovell, Hisoka-X, Holden Karau, Huanli Wang, Hyukjin Kwon, Ismaël Mejía, Ivan Sadikov, Jack Chen, James Lamb, Jason Li, Jerry Peng, Jia Fan, Jiaan Geng, JinHelin404, Joe Wang, Johan Lasperas, John Zhuge, Josh Rosen, Juliusz Sompolski, Jungtaek Lim, Kazuyuki Tanimura, Kent Yao, Khalid Mammadov, Koray Beyaz, Kris Mok, Kun Wan, Kwafoor, Lee Yang, Leibnizhu, Liang Yan, Liang-Chi Hsieh, Lingkai Kong, Luca Canali, Ludovic Henry, Manu Zhang, Martin Grund, Mathew Jacob, Max Gekk, Menelaos Karavelas, Michael Chen, Michael Zhang, Mingkang Li, Mridul Muralidharan, NarekDW, Navin Viswanath, Niranjan Jayakar, Ole Sasse, Parth Upadhyay, Peter Toth, Qi Tan, Raghu Angadi, Richard Yu, Ritika Maheshwari, Rob Reeves, Robert Dillitz, Rui Wang, Ruifeng Zheng, Runyao Chen, Ryan Berti, Ryan Johnson, Serge Rielau, Serge Smertin, Sergii Druzkin, Shaoyun Chen, Shreyesh Shaju Arangath, Shrikant Prasad, Shu Wang, Siying Dong, Steve Vaughan Jr, Steve Weis, Steven Aerts, Steven Chen, Stove-hust, Szehon Ho, Takuya UESHIN, Tengfei Huang, Terry Kim, Thejdeep Gudivada, Thomas Graves, Tim Nieradzik, Venki Korukanti, Vihang Karajgaonkar, Vinod KC, Vitalii Li, Vsevolod Stepanov, Wanqiang Ji, Warren Zhu, Wei Liu, Weichen Xu, Wen Yuen Pang, Wenchen Fan, Willi Raschkowski, William Hyun, Xiduo You, Xieming LI, Xingbo Jiang, Xinrong Meng, Xinyi Yu, Yang Jie, Yi Zhu, Yihong He, Yikf, Yikun Jiang, Yiqun Zhang, Yohahaha, Yuanjian Li, Yuming Wang, Zhen Li, Ziqi Liu, advancedxy, alexanderwu-db, amousavigourabi, bjornjorgensen, bogao007, bowenliang123, caican00, chong0929, clownxc, frankliee, haoyanzhang, hdaly0, huangxiaopingRD, jdesjean, jerqi, jwang0306, khalidmammadov, kings129, kylerong-db, liangyu-1, lucaspompeun, mcdull-zhang, panbingkun, pegasas, pengzhon-db, ronandoolan2, smallzhongfeng, srielau, stijndehaes, sudoliyang, ted-jenks, tianhanhu, vicennial, wForget, wayne-kyungwonpark, wayneguow, yabola, zeruibao, zhouyifan279, zml1206, zzzzming95 + + diff --git a/site/committers.html b/site/committers.html index d54c96e9314..33206e99947 100644 --- a/site/committers.html +++ b/site/committers.html @@ -657,6 +657,9 @@

    Policy on backporting bug fixes

    Latest News

    Archive

    diff --git a/site/community.html b/site/community.html index 1b649e898af..6c73a8f516c 100644 --- a/site/community.html +++ b/site/community.html @@ -348,6 +348,9 @@

    Powered by

    Latest News

    Archive

    diff --git a/site/contributing.html b/site/contributing.html index 65f85bae275..6b76d553ffb 100644 --- a/site/contributing.html +++ b/site/contributing.html @@ -658,6 +658,9 @@

    Code of conduct

    Latest News

    Archive

    diff --git a/site/developer-tools.html b/site/developer-tools.html index 334195939d8..5690ede14fc 100644 --- a/site/developer-tools.html +++ b/site/developer-tools.html @@ -672,6 +672,9 @@

    Generative tooling usage

    Latest News

    Archive

    diff --git a/site/docs/latest b/site/docs/latest index 8cf6caf561b..e5b820341fb 120000 --- a/site/docs/latest +++ b/site/docs/latest @@ -1 +1 @@ -3.4.1 \ No newline at end of file +3.5.0 \ No newline at end of file diff --git a/site/documentation.html b/site/documentation.html index f25b274e651..087665cfa40 100644 --- a/site/documentation.html +++ b/site/documentation.html @@ -127,6 +127,7 @@

    Apache Spark™Setup instructions, programming guides, and other documentation are available for each stable version of Spark below:

      +
    • Spark 3.5.0
    • Spark 3.4.1
    • Spark 3.4.0
    • Spark 3.3.3
      • @@ -340,6 +341,9 @@

      Research Papers

      Latest News

      Archive

      diff --git a/site/downloads.html b/site/downloads.html index 62319c427b6..f87f6fe1ae0 100644 --- a/site/downloads.html +++ b/site/downloads.html @@ -157,7 +157,7 @@ 
      groupId: org.apache.spark
 artifactId: spark-core_2.12
-version: 3.4.1
+version: 3.5.0

      Installing with PyPi

      @@ -186,6 +186,9 @@

      Archived releases

      Latest News

      Archive

      diff --git a/site/error-message-guidelines.html b/site/error-message-guidelines.html index 50342288a5e..f608642bcac 100644 --- a/site/error-message-guidelines.html +++ b/site/error-message-guidelines.html @@ -525,6 +525,9 @@

      Wording guide

      Latest News

      Archive

      diff --git a/site/examples.html b/site/examples.html index eab8ac9e709..34713807ae0 100644 --- a/site/examples.html +++ b/site/examples.html @@ -527,6 +527,9 @@

      Additional examples

      Latest News

      Archive

      diff --git a/site/faq.html b/site/faq.html index 0b2212a068e..fff0138d9ea 100644 --- a/site/faq.html +++ b/site/faq.html @@ -195,6 +195,9 @@

      Apache Spark™Latest News

      Archive

      diff --git a/site/graphx/index.html b/site/graphx/index.html index 806033154dd..a7fb0c1934b 100644 --- a/site/graphx/index.html +++ b/site/graphx/index.html @@ -240,6 +240,9 @@

      Getting started

      Latest News

      Archive

      diff --git a/site/history.html b/site/history.html index ebba874fe46..a72277909b2 100644 --- a/site/history.html +++ b/site/history.html @@ -149,6 +149,9 @@

      Apache Spark™Latest News

      Archive

      diff --git a/site/improvement-proposals.html b/site/improvement-proposals.html index 25dd3eade28..1f2f815d621 100644 --- a/site/improvement-proposals.html +++ b/site/improvement-proposals.html @@ -221,6 +221,9 @@

      Implementing an SPIP

      Latest News

      Archive

      diff --git a/site/js/downloads.js b/site/js/downloads.js index 193b32f8cfd..5a76dbd9742 100644 --- a/site/js/downloads.js +++ b/site/js/downloads.js @@ -27,6 +27,7 @@ var packagesV13 = [hadoop3p, hadoop3pscala213, hadoop2p, hadoopFree, sources]; // 3.4.0+ var packagesV14 = [hadoop3p, hadoop3pscala213, hadoopFree, sources]; +addRelease("3.5.0", new Date("09/13/2023"), packagesV14, true); addRelease("3.4.1", new Date("06/23/2023"), packagesV14, true); addRelease("3.3.3", new Date("08/21/2023"), packagesV13, true); addRelease("3.2.4", new Date("04/13/2023"), packagesV12, true); diff --git a/site/mailing-lists.html b/site/mailing-lists.html index 4f584d29592..0430ee17c3a 100644 --- a/site/mailing-lists.html +++ b/site/mailing-lists.html @@ -133,6 +133,9 @@
      Latest News

      Archive

      diff --git a/site/mllib/index.html b/site/mllib/index.html index 05807e3dc21..65485c7cefc 100644 --- a/site/mllib/index.html +++ b/site/mllib/index.html @@ -286,6 +286,9 @@

      Getting started

      Latest News

      Archive

      diff --git a/site/news/3-1-3-released.html b/site/news/3-1-3-released.html index 9e8414524e6..e9c5e2132d1 100644 --- a/site/news/3-1-3-released.html +++ b/site/news/3-1-3-released.html @@ -139,6 +139,9 @@

      Spark 3.1.3 released

      Latest News

      Archive

      diff --git a/site/news/amp-camp-2013-registration-ope.html b/site/news/amp-camp-2013-registration-ope.html index 596d6d6cf92..1533e36abfe 100644 --- a/site/news/amp-camp-2013-registration-ope.html +++ b/site/news/amp-camp-2013-registration-ope.html @@ -139,6 +139,9 @@

      Registration open for AMP Camp training camp in Berkeley

      Latest News

      Archive

      diff --git a/site/news/announcing-the-first-spark-summit.html b/site/news/announcing-the-first-spark-summit.html index 77de32cd625..f372710189c 100644 --- a/site/news/announcing-the-first-spark-summit.html +++ b/site/news/announcing-the-first-spark-summit.html @@ -143,6 +143,9 @@

      Announcing the first Spark Summit: December 2, 2013

      Latest News

      Archive

      diff --git a/site/news/fourth-spark-screencast-published.html b/site/news/fourth-spark-screencast-published.html index 68983f90434..880b03eaac9 100644 --- a/site/news/fourth-spark-screencast-published.html +++ b/site/news/fourth-spark-screencast-published.html @@ -143,6 +143,9 @@

      Fourth Spark screencast released

      Latest News

      Archive

      diff --git a/site/news/index.html b/site/news/index.html index ca7cb02638a..f3ba2ccc22c 100644 --- a/site/news/index.html +++ b/site/news/index.html @@ -124,6 +124,15 @@

      Spark News

      + +
      diff --git a/site/news/new-repository-service.html b/site/news/new-repository-service.html index 05c810f867f..b87ceeb98b2 100644 --- a/site/news/new-repository-service.html +++ b/site/news/new-repository-service.html @@ -145,6 +145,9 @@

      New repository service for spark-packages

      Latest News

      Archive

      diff --git a/site/news/next-official-release-spark-3.1.1.html b/site/news/next-official-release-spark-3.1.1.html index d0f407ed266..723ce6119bc 100644 --- a/site/news/next-official-release-spark-3.1.1.html +++ b/site/news/next-official-release-spark-3.1.1.html @@ -151,6 +151,9 @@

      Next official release: Spark 3.1.1

      Latest News

      Archive

      diff --git a/site/news/nsdi-paper.html b/site/news/nsdi-paper.html index bb56bc3b5ca..a07e22d370a 100644 --- a/site/news/nsdi-paper.html +++ b/site/news/nsdi-paper.html @@ -139,6 +139,9 @@

      Spark wins Best Paper Award at USENIX NSDI

      Latest News

      Archive

      diff --git a/site/news/one-month-to-spark-summit-2015.html b/site/news/one-month-to-spark-summit-2015.html index f5f29f931b1..fe72dff88d4 100644 --- a/site/news/one-month-to-spark-summit-2015.html +++ b/site/news/one-month-to-spark-summit-2015.html @@ -145,6 +145,9 @@

      One month to Spark Summit 2015 in San Francisco

      Latest News

      Archive

      diff --git a/site/news/plan-for-dropping-python-2-support.html b/site/news/plan-for-dropping-python-2-support.html index 71f53d49961..ec3b8f00a97 100644 --- a/site/news/plan-for-dropping-python-2-support.html +++ b/site/news/plan-for-dropping-python-2-support.html @@ -155,6 +155,9 @@

      Plan for dropping Python 2 support

      Latest News

      Archive

      diff --git a/site/news/proposals-open-for-spark-summit-east.html b/site/news/proposals-open-for-spark-summit-east.html index 84fbd9da17a..0f9186ec9fc 100644 --- a/site/news/proposals-open-for-spark-summit-east.html +++ b/site/news/proposals-open-for-spark-summit-east.html @@ -143,6 +143,9 @@

      Submissions open for Spark Summit East 2015 in New York

      Latest News

      Archive

      diff --git a/site/news/registration-open-for-spark-summit-east.html b/site/news/registration-open-for-spark-summit-east.html index eb79696a1ba..31ea969843c 100644 --- a/site/news/registration-open-for-spark-summit-east.html +++ b/site/news/registration-open-for-spark-summit-east.html @@ -143,6 +143,9 @@

      Registration open for Spark Summit East 2015

      Latest News

      Archive

      diff --git a/site/news/run-spark-and-shark-on-amazon-emr.html b/site/news/run-spark-and-shark-on-amazon-emr.html index 5a1fd95dc53..32783caff5c 100644 --- a/site/news/run-spark-and-shark-on-amazon-emr.html +++ b/site/news/run-spark-and-shark-on-amazon-emr.html @@ -139,6 +139,9 @@

      Spark/Shark Tutorial for Amazon EMR

      Latest News

      Archive

      diff --git a/site/news/sigmod-system-award.html b/site/news/sigmod-system-award.html index f23376b85cf..ad900234d11 100644 --- a/site/news/sigmod-system-award.html +++ b/site/news/sigmod-system-award.html @@ -150,6 +150,9 @@

      SIGMOD Systems Award for Apache Spark

      Latest News

      Archive

      diff --git a/site/news/spark-0-6-1-and-0-5-2-released.html b/site/news/spark-0-6-1-and-0-5-2-released.html index b4447734a70..dc6fed728d3 100644 --- a/site/news/spark-0-6-1-and-0-5-2-released.html +++ b/site/news/spark-0-6-1-and-0-5-2-released.html @@ -139,6 +139,9 @@

      Spark 0.6.1 and 0.5.2 out

      Latest News

      Archive

      diff --git a/site/news/spark-0-6-2-released.html b/site/news/spark-0-6-2-released.html index 69985bc4f7b..477b764b75b 100644 --- a/site/news/spark-0-6-2-released.html +++ b/site/news/spark-0-6-2-released.html @@ -139,6 +139,9 @@

      Spark 0.6.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-7-0-released.html b/site/news/spark-0-7-0-released.html index 7138aca5f78..6125cb299d6 100644 --- a/site/news/spark-0-7-0-released.html +++ b/site/news/spark-0-7-0-released.html @@ -139,6 +139,9 @@

      Spark 0.7.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-7-2-released.html b/site/news/spark-0-7-2-released.html index 5c67506c8aa..93da9f07d9a 100644 --- a/site/news/spark-0-7-2-released.html +++ b/site/news/spark-0-7-2-released.html @@ -139,6 +139,9 @@

      Spark 0.7.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-7-3-released.html b/site/news/spark-0-7-3-released.html index d94a9b5187a..0cff9ad2cd4 100644 --- a/site/news/spark-0-7-3-released.html +++ b/site/news/spark-0-7-3-released.html @@ -139,6 +139,9 @@

      Spark 0.7.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-8-0-released.html b/site/news/spark-0-8-0-released.html index 3ef4c73a0c7..9ab1f0ebac7 100644 --- a/site/news/spark-0-8-0-released.html +++ b/site/news/spark-0-8-0-released.html @@ -139,6 +139,9 @@

      Spark 0.8.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-8-1-released.html b/site/news/spark-0-8-1-released.html index c3b4dda3667..ab45bf88dfb 100644 --- a/site/news/spark-0-8-1-released.html +++ b/site/news/spark-0-8-1-released.html @@ -139,6 +139,9 @@

      Spark 0.8.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-9-0-released.html b/site/news/spark-0-9-0-released.html index ff9885044b2..3060e3d2f0c 100644 --- a/site/news/spark-0-9-0-released.html +++ b/site/news/spark-0-9-0-released.html @@ -146,6 +146,9 @@

      Spark 0.9.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-9-1-released.html b/site/news/spark-0-9-1-released.html index 060a9037a60..15114101f60 100644 --- a/site/news/spark-0-9-1-released.html +++ b/site/news/spark-0-9-1-released.html @@ -145,6 +145,9 @@

      Spark 0.9.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-0-9-2-released.html b/site/news/spark-0-9-2-released.html index 368e7aeb137..030ec6f162b 100644 --- a/site/news/spark-0-9-2-released.html +++ b/site/news/spark-0-9-2-released.html @@ -144,6 +144,9 @@

      Spark 0.9.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-0-0-released.html b/site/news/spark-1-0-0-released.html index 4c70de31ee9..b1d730d142a 100644 --- a/site/news/spark-1-0-0-released.html +++ b/site/news/spark-1-0-0-released.html @@ -142,6 +142,9 @@

      Spark 1.0.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-0-1-released.html b/site/news/spark-1-0-1-released.html index ba8719d4d38..e63f2e335a2 100644 --- a/site/news/spark-1-0-1-released.html +++ b/site/news/spark-1-0-1-released.html @@ -141,6 +141,9 @@

      Spark 1.0.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-0-2-released.html b/site/news/spark-1-0-2-released.html index decbe49737f..003ec35e94a 100644 --- a/site/news/spark-1-0-2-released.html +++ b/site/news/spark-1-0-2-released.html @@ -141,6 +141,9 @@

      Spark 1.0.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-1-0-released.html b/site/news/spark-1-1-0-released.html index c9698692bb2..33d6b3e0cc0 100644 --- a/site/news/spark-1-1-0-released.html +++ b/site/news/spark-1-1-0-released.html @@ -143,6 +143,9 @@

      Spark 1.1.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-1-1-released.html b/site/news/spark-1-1-1-released.html index fb815cbc98a..ff642e0e29b 100644 --- a/site/news/spark-1-1-1-released.html +++ b/site/news/spark-1-1-1-released.html @@ -141,6 +141,9 @@

      Spark 1.1.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-2-0-released.html b/site/news/spark-1-2-0-released.html index ccee9b14947..152bf4b2353 100644 --- a/site/news/spark-1-2-0-released.html +++ b/site/news/spark-1-2-0-released.html @@ -143,6 +143,9 @@

      Spark 1.2.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-2-1-released.html b/site/news/spark-1-2-1-released.html index 95290c05ba8..b47ea0d395a 100644 --- a/site/news/spark-1-2-1-released.html +++ b/site/news/spark-1-2-1-released.html @@ -141,6 +141,9 @@

      Spark 1.2.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-2-2-released.html b/site/news/spark-1-2-2-released.html index 269ec3665ef..9209012f2cf 100644 --- a/site/news/spark-1-2-2-released.html +++ b/site/news/spark-1-2-2-released.html @@ -141,6 +141,9 @@

      Spark 1.2.2 and 1.3.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-3-0-released.html b/site/news/spark-1-3-0-released.html index 476e931ece1..a27224a0765 100644 --- a/site/news/spark-1-3-0-released.html +++ b/site/news/spark-1-3-0-released.html @@ -141,6 +141,9 @@

      Spark 1.3.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-4-0-released.html b/site/news/spark-1-4-0-released.html index bdf94aa100f..fabc553901f 100644 --- a/site/news/spark-1-4-0-released.html +++ b/site/news/spark-1-4-0-released.html @@ -141,6 +141,9 @@

      Spark 1.4.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-4-1-released.html b/site/news/spark-1-4-1-released.html index 3a7a191f19d..2f25543aa18 100644 --- a/site/news/spark-1-4-1-released.html +++ b/site/news/spark-1-4-1-released.html @@ -141,6 +141,9 @@

      Spark 1.4.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-5-0-released.html b/site/news/spark-1-5-0-released.html index c582c93d274..fe7f814301a 100644 --- a/site/news/spark-1-5-0-released.html +++ b/site/news/spark-1-5-0-released.html @@ -141,6 +141,9 @@

      Spark 1.5.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-5-1-released.html b/site/news/spark-1-5-1-released.html index d58bdb39e44..0324fa3287d 100644 --- a/site/news/spark-1-5-1-released.html +++ b/site/news/spark-1-5-1-released.html @@ -141,6 +141,9 @@

      Spark 1.5.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-5-2-released.html b/site/news/spark-1-5-2-released.html index 77602640da1..ae1a1d67a70 100644 --- a/site/news/spark-1-5-2-released.html +++ b/site/news/spark-1-5-2-released.html @@ -141,6 +141,9 @@

      Spark 1.5.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-6-0-released.html b/site/news/spark-1-6-0-released.html index 09cbcdc0e0a..319f5b58fb0 100644 --- a/site/news/spark-1-6-0-released.html +++ b/site/news/spark-1-6-0-released.html @@ -145,6 +145,9 @@

      Spark 1.6.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-6-1-released.html b/site/news/spark-1-6-1-released.html index 4fc81a2ea64..92a0e808c94 100644 --- a/site/news/spark-1-6-1-released.html +++ b/site/news/spark-1-6-1-released.html @@ -141,6 +141,9 @@

      Spark 1.6.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-6-2-released.html b/site/news/spark-1-6-2-released.html index 2b5f3d8a15f..963df6b8489 100644 --- a/site/news/spark-1-6-2-released.html +++ b/site/news/spark-1-6-2-released.html @@ -141,6 +141,9 @@

      Spark 1.6.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-1-6-3-released.html b/site/news/spark-1-6-3-released.html index bc3f0da3736..d9100dababd 100644 --- a/site/news/spark-1-6-3-released.html +++ b/site/news/spark-1-6-3-released.html @@ -141,6 +141,9 @@

      Spark 1.6.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-0-0-released.html b/site/news/spark-2-0-0-released.html index e1a97a8742c..84c6a7e0433 100644 --- a/site/news/spark-2-0-0-released.html +++ b/site/news/spark-2-0-0-released.html @@ -139,6 +139,9 @@

      Spark 2.0.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-0-1-released.html b/site/news/spark-2-0-1-released.html index 6b2fd95b343..0ae708f944d 100644 --- a/site/news/spark-2-0-1-released.html +++ b/site/news/spark-2-0-1-released.html @@ -139,6 +139,9 @@

      Spark 2.0.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-0-2-released.html b/site/news/spark-2-0-2-released.html index c716061534f..b0e121748bd 100644 --- a/site/news/spark-2-0-2-released.html +++ b/site/news/spark-2-0-2-released.html @@ -141,6 +141,9 @@

      Spark 2.0.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-1-0-released.html b/site/news/spark-2-1-0-released.html index 2fa24130f1f..7e6d5ae507a 100644 --- a/site/news/spark-2-1-0-released.html +++ b/site/news/spark-2-1-0-released.html @@ -139,6 +139,9 @@

      Spark 2.1.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-1-1-released.html b/site/news/spark-2-1-1-released.html index 3bddb368813..896b92512fc 100644 --- a/site/news/spark-2-1-1-released.html +++ b/site/news/spark-2-1-1-released.html @@ -140,6 +140,9 @@

      Spark 2.1.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-1-2-released.html b/site/news/spark-2-1-2-released.html index 9f76c5ae66c..6511dabdbf9 100644 --- a/site/news/spark-2-1-2-released.html +++ b/site/news/spark-2-1-2-released.html @@ -140,6 +140,9 @@

      Spark 2.1.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-1-3-released.html b/site/news/spark-2-1-3-released.html index ee1400841db..296afc95222 100644 --- a/site/news/spark-2-1-3-released.html +++ b/site/news/spark-2-1-3-released.html @@ -139,6 +139,9 @@

      Spark 2.1.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-2-0-released.html b/site/news/spark-2-2-0-released.html index 81f2cfc45ce..c35ed967207 100644 --- a/site/news/spark-2-2-0-released.html +++ b/site/news/spark-2-2-0-released.html @@ -139,6 +139,9 @@

      Spark 2.2.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-2-1-released.html b/site/news/spark-2-2-1-released.html index ce3bc5068f7..089d5e00768 100644 --- a/site/news/spark-2-2-1-released.html +++ b/site/news/spark-2-2-1-released.html @@ -140,6 +140,9 @@

      Spark 2.2.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-2-2-released.html b/site/news/spark-2-2-2-released.html index 2386c087508..e9c84d37b80 100644 --- a/site/news/spark-2-2-2-released.html +++ b/site/news/spark-2-2-2-released.html @@ -139,6 +139,9 @@

      Spark 2.2.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-3-0-released.html b/site/news/spark-2-3-0-released.html index fb2fd403ff4..314af7ba9ec 100644 --- a/site/news/spark-2-3-0-released.html +++ b/site/news/spark-2-3-0-released.html @@ -139,6 +139,9 @@

      Spark 2.3.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-3-1-released.html b/site/news/spark-2-3-1-released.html index 5f813c5f1eb..25506938a77 100644 --- a/site/news/spark-2-3-1-released.html +++ b/site/news/spark-2-3-1-released.html @@ -139,6 +139,9 @@

      Spark 2.3.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-3-2-released.html b/site/news/spark-2-3-2-released.html index 2ff329920d4..ef8fbfbaf7e 100644 --- a/site/news/spark-2-3-2-released.html +++ b/site/news/spark-2-3-2-released.html @@ -139,6 +139,9 @@

      Spark 2.3.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-3-3-released.html b/site/news/spark-2-3-3-released.html index 5ded7c85624..639ed935657 100644 --- a/site/news/spark-2-3-3-released.html +++ b/site/news/spark-2-3-3-released.html @@ -139,6 +139,9 @@

      Spark 2.3.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-3-4-released.html b/site/news/spark-2-3-4-released.html index e786aa5b9ef..57413554cec 100644 --- a/site/news/spark-2-3-4-released.html +++ b/site/news/spark-2-3-4-released.html @@ -139,6 +139,9 @@

      Spark 2.3.4 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-0-released.html b/site/news/spark-2-4-0-released.html index 92a6f59eacd..df61732406f 100644 --- a/site/news/spark-2-4-0-released.html +++ b/site/news/spark-2-4-0-released.html @@ -139,6 +139,9 @@

      Spark 2.4.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-1-released.html b/site/news/spark-2-4-1-released.html index 4318d5a4217..b037525dafd 100644 --- a/site/news/spark-2-4-1-released.html +++ b/site/news/spark-2-4-1-released.html @@ -140,6 +140,9 @@

      Spark 2.4.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-2-released.html b/site/news/spark-2-4-2-released.html index 83d4f8e367f..4b6cbd9eb6e 100644 --- a/site/news/spark-2-4-2-released.html +++ b/site/news/spark-2-4-2-released.html @@ -139,6 +139,9 @@

      Spark 2.4.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-3-released.html b/site/news/spark-2-4-3-released.html index a619c0f9df3..e4599d43583 100644 --- a/site/news/spark-2-4-3-released.html +++ b/site/news/spark-2-4-3-released.html @@ -139,6 +139,9 @@

      Spark 2.4.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-4-released.html b/site/news/spark-2-4-4-released.html index d0694823a60..43fdbc897b8 100644 --- a/site/news/spark-2-4-4-released.html +++ b/site/news/spark-2-4-4-released.html @@ -139,6 +139,9 @@

      Spark 2.4.4 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-5-released.html b/site/news/spark-2-4-5-released.html index 7465504ad51..c8c01a6fd56 100644 --- a/site/news/spark-2-4-5-released.html +++ b/site/news/spark-2-4-5-released.html @@ -139,6 +139,9 @@

      Spark 2.4.5 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-6.html b/site/news/spark-2-4-6.html index 8a1d1c3af91..180a39cc50e 100644 --- a/site/news/spark-2-4-6.html +++ b/site/news/spark-2-4-6.html @@ -139,6 +139,9 @@

      Spark 2.4.6 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-7-released.html b/site/news/spark-2-4-7-released.html index 199534a5b03..f9a2ab81b8e 100644 --- a/site/news/spark-2-4-7-released.html +++ b/site/news/spark-2-4-7-released.html @@ -139,6 +139,9 @@

      Spark 2.4.7 released

      Latest News

      Archive

      diff --git a/site/news/spark-2-4-8-released.html b/site/news/spark-2-4-8-released.html index 720763c8d63..6d6b4c3520b 100644 --- a/site/news/spark-2-4-8-released.html +++ b/site/news/spark-2-4-8-released.html @@ -139,6 +139,9 @@

      Spark 2.4.8 released

      Latest News

      Archive

      diff --git a/site/news/spark-2.0.0-preview.html b/site/news/spark-2.0.0-preview.html index 9aef0992b69..a6a123203ab 100644 --- a/site/news/spark-2.0.0-preview.html +++ b/site/news/spark-2.0.0-preview.html @@ -141,6 +141,9 @@

      Preview release of Spark 2.0

      Latest News

      Archive

      diff --git a/site/news/spark-3-0-0-released.html b/site/news/spark-3-0-0-released.html index 24943a8f178..1cecf8351fb 100644 --- a/site/news/spark-3-0-0-released.html +++ b/site/news/spark-3-0-0-released.html @@ -139,6 +139,9 @@

      Spark 3.0.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-0-1-released.html b/site/news/spark-3-0-1-released.html index 365cb0b0cb2..800468d733b 100644 --- a/site/news/spark-3-0-1-released.html +++ b/site/news/spark-3-0-1-released.html @@ -139,6 +139,9 @@

      Spark 3.0.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-0-2-released.html b/site/news/spark-3-0-2-released.html index d03652b7ddb..1fdc43cae3b 100644 --- a/site/news/spark-3-0-2-released.html +++ b/site/news/spark-3-0-2-released.html @@ -139,6 +139,9 @@

      Spark 3.0.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-0-3-released.html b/site/news/spark-3-0-3-released.html index e8569f8cae3..4edf7df8081 100644 --- a/site/news/spark-3-0-3-released.html +++ b/site/news/spark-3-0-3-released.html @@ -139,6 +139,9 @@

      Spark 3.0.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-1-1-released.html b/site/news/spark-3-1-1-released.html index 78e249fa1d1..dd88af46514 100644 --- a/site/news/spark-3-1-1-released.html +++ b/site/news/spark-3-1-1-released.html @@ -139,6 +139,9 @@

      Spark 3.1.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-1-2-released.html b/site/news/spark-3-1-2-released.html index 007fcc65f98..4b0e4244d79 100644 --- a/site/news/spark-3-1-2-released.html +++ b/site/news/spark-3-1-2-released.html @@ -139,6 +139,9 @@

      Spark 3.1.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-2-0-released.html b/site/news/spark-3-2-0-released.html index 26446e92f0f..48993cbe37c 100644 --- a/site/news/spark-3-2-0-released.html +++ b/site/news/spark-3-2-0-released.html @@ -139,6 +139,9 @@

      Spark 3.2.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-2-1-released.html b/site/news/spark-3-2-1-released.html index d7e7e130181..16217149102 100644 --- a/site/news/spark-3-2-1-released.html +++ b/site/news/spark-3-2-1-released.html @@ -139,6 +139,9 @@

      Spark 3.2.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-2-2-released.html b/site/news/spark-3-2-2-released.html index ff474bd5575..088e9f44e57 100644 --- a/site/news/spark-3-2-2-released.html +++ b/site/news/spark-3-2-2-released.html @@ -139,6 +139,9 @@

      Spark 3.2.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-2-3-released.html b/site/news/spark-3-2-3-released.html index 955e6ac4ef3..9c19ee8d232 100644 --- a/site/news/spark-3-2-3-released.html +++ b/site/news/spark-3-2-3-released.html @@ -139,6 +139,9 @@

      Spark 3.2.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-2-4-released.html b/site/news/spark-3-2-4-released.html index 068dc9f6b94..73a9e995df6 100644 --- a/site/news/spark-3-2-4-released.html +++ b/site/news/spark-3-2-4-released.html @@ -139,6 +139,9 @@

      Spark 3.2.4 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-3-0-released.html b/site/news/spark-3-3-0-released.html index 0a693787765..0b8ba8a7e7c 100644 --- a/site/news/spark-3-3-0-released.html +++ b/site/news/spark-3-3-0-released.html @@ -139,6 +139,9 @@

      Spark 3.3.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-3-1-released.html b/site/news/spark-3-3-1-released.html index c9918333e86..5e377fb94b6 100644 --- a/site/news/spark-3-3-1-released.html +++ b/site/news/spark-3-3-1-released.html @@ -139,6 +139,9 @@

      Spark 3.3.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-3-2-released.html b/site/news/spark-3-3-2-released.html index 3aa660d61ed..463453c6c33 100644 --- a/site/news/spark-3-3-2-released.html +++ b/site/news/spark-3-3-2-released.html @@ -139,6 +139,9 @@

      Spark 3.3.2 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-3-3-released.html b/site/news/spark-3-3-3-released.html index e751160afa6..502649e6131 100644 --- a/site/news/spark-3-3-3-released.html +++ b/site/news/spark-3-3-3-released.html @@ -139,6 +139,9 @@

      Spark 3.3.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-4-0-released.html b/site/news/spark-3-4-0-released.html index ce7d680a232..b2d197004ae 100644 --- a/site/news/spark-3-4-0-released.html +++ b/site/news/spark-3-4-0-released.html @@ -139,6 +139,9 @@

      Spark 3.4.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-4-1-released.html b/site/news/spark-3-4-1-released.html index e2e64e6e903..f315148cf08 100644 --- a/site/news/spark-3-4-1-released.html +++ b/site/news/spark-3-4-1-released.html @@ -139,6 +139,9 @@

      Spark 3.4.1 released

      Latest News

      Archive

      diff --git a/site/news/spark-3-5-0-released.html b/site/news/spark-3-5-0-released.html new file mode 100644 index 00000000000..cc50c369aba --- /dev/null +++ b/site/news/spark-3-5-0-released.html @@ -0,0 +1,200 @@ + + + + + + + + + Spark 3.5.0 released | Apache Spark + + + + + + + + + + + + + + + + + + + +
      +
      +
      +

      Spark 3.5.0 released

      + + +

      We are happy to announce the availability of Spark 3.5.0! Visit the release notes to read about the new features, or download the release today.

      + + +

      +
      +Spark News Archive +

      + +
      +
      +
      +
      Latest News
      + +

      Archive

      +
      +
      + + + +
      + +
      +
      + + + +
      +
      + Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered + trademarks or trademarks of The Apache Software Foundation in the United States and other countries. + See guidance on use of Apache Spark trademarks. + All other marks mentioned may be trademarks or registered trademarks of their respective owners. + Copyright © 2018 The Apache Software Foundation, Licensed under the + Apache License, Version 2.0. +
      +
      + + + + + + + diff --git a/site/news/spark-3.0.0-preview.html b/site/news/spark-3.0.0-preview.html index 6a4f97ed001..db03a730b53 100644 --- a/site/news/spark-3.0.0-preview.html +++ b/site/news/spark-3.0.0-preview.html @@ -141,6 +141,9 @@

      Preview release of Spark 3.0

      Latest News

      Archive

      diff --git a/site/news/spark-3.0.0-preview2.html b/site/news/spark-3.0.0-preview2.html index edb55085a5c..bd26aacb0d9 100644 --- a/site/news/spark-3.0.0-preview2.html +++ b/site/news/spark-3.0.0-preview2.html @@ -141,6 +141,9 @@

      Preview release of Spark 3.0

      Latest News

      Archive

      diff --git a/site/news/spark-accepted-into-apache-incubator.html b/site/news/spark-accepted-into-apache-incubator.html index 1401d05db87..0fec1c674eb 100644 --- a/site/news/spark-accepted-into-apache-incubator.html +++ b/site/news/spark-accepted-into-apache-incubator.html @@ -139,6 +139,9 @@

      Spark accepted into Apache Incubator

      Latest News

      Archive

      diff --git a/site/news/spark-ai-summit-apr-2019-agenda-posted.html b/site/news/spark-ai-summit-apr-2019-agenda-posted.html index 9b85b738502..4c405077f0a 100644 --- a/site/news/spark-ai-summit-apr-2019-agenda-posted.html +++ b/site/news/spark-ai-summit-apr-2019-agenda-posted.html @@ -139,6 +139,9 @@

      Spark+AI Summit (April 23-25th, 2019, San Francisco) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-ai-summit-june-2020-agenda-posted.html b/site/news/spark-ai-summit-june-2020-agenda-posted.html index 4ee047e0469..7c5a0068512 100644 --- a/site/news/spark-ai-summit-june-2020-agenda-posted.html +++ b/site/news/spark-ai-summit-june-2020-agenda-posted.html @@ -139,6 +139,9 @@

      Spark+AI Summit (June 22-25th, 2020, VIRTUAL) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-and-shark-in-the-news.html b/site/news/spark-and-shark-in-the-news.html index 9f77fcbd4fa..64d02e65c18 100644 --- a/site/news/spark-and-shark-in-the-news.html +++ b/site/news/spark-and-shark-in-the-news.html @@ -149,6 +149,9 @@

      Spark in the news

      Latest News

      Archive

      diff --git a/site/news/spark-becomes-tlp.html b/site/news/spark-becomes-tlp.html index 28471571c5e..e8e86012ddc 100644 --- a/site/news/spark-becomes-tlp.html +++ b/site/news/spark-becomes-tlp.html @@ -141,6 +141,9 @@

      Spark becomes top-level Apache project

      Latest News

      Archive

      diff --git a/site/news/spark-featured-in-wired.html b/site/news/spark-featured-in-wired.html index 6c0503763b6..eb1b1314f34 100644 --- a/site/news/spark-featured-in-wired.html +++ b/site/news/spark-featured-in-wired.html @@ -139,6 +139,9 @@

      Spark featured in Wired

      Latest News

      Archive

      diff --git a/site/news/spark-mailing-lists-moving-to-apache.html b/site/news/spark-mailing-lists-moving-to-apache.html index adc8891f676..c73d5f30520 100644 --- a/site/news/spark-mailing-lists-moving-to-apache.html +++ b/site/news/spark-mailing-lists-moving-to-apache.html @@ -150,6 +150,9 @@

      Spark mailing lists moving to Apache

      Latest News

      Archive

      diff --git a/site/news/spark-meetups.html b/site/news/spark-meetups.html index cf7c25663f7..a7360050cf7 100644 --- a/site/news/spark-meetups.html +++ b/site/news/spark-meetups.html @@ -139,6 +139,9 @@

      We've started hosting a Bay Area Spark User Meetup

      Latest News

      Archive

      diff --git a/site/news/spark-release-2-2-3.html b/site/news/spark-release-2-2-3.html index 47b223a01b4..710a5a7365a 100644 --- a/site/news/spark-release-2-2-3.html +++ b/site/news/spark-release-2-2-3.html @@ -139,6 +139,9 @@

      Spark 2.2.3 released

      Latest News

      Archive

      diff --git a/site/news/spark-screencasts-published.html b/site/news/spark-screencasts-published.html index 504bada5017..55244faf81f 100644 --- a/site/news/spark-screencasts-published.html +++ b/site/news/spark-screencasts-published.html @@ -145,6 +145,9 @@

      Spark screencasts published

      Latest News

      Archive

      diff --git a/site/news/spark-summit-2013-is-a-wrap.html b/site/news/spark-summit-2013-is-a-wrap.html index b589635f645..379d050d49b 100644 --- a/site/news/spark-summit-2013-is-a-wrap.html +++ b/site/news/spark-summit-2013-is-a-wrap.html @@ -145,6 +145,9 @@

      Spark Summit 2013 is a Wrap

      Latest News

      Archive

      diff --git a/site/news/spark-summit-2014-videos-posted.html b/site/news/spark-summit-2014-videos-posted.html index e926246d920..0b6ef90899f 100644 --- a/site/news/spark-summit-2014-videos-posted.html +++ b/site/news/spark-summit-2014-videos-posted.html @@ -141,6 +141,9 @@

      Spark Summit 2014 videos posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-2015-videos-posted.html b/site/news/spark-summit-2015-videos-posted.html index 8c69d8a5d29..3004825f99c 100644 --- a/site/news/spark-summit-2015-videos-posted.html +++ b/site/news/spark-summit-2015-videos-posted.html @@ -139,6 +139,9 @@

      Spark Summit 2015 Videos Posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-agenda-posted.html b/site/news/spark-summit-agenda-posted.html index 467eabc3892..1b98c6f34d4 100644 --- a/site/news/spark-summit-agenda-posted.html +++ b/site/news/spark-summit-agenda-posted.html @@ -147,6 +147,9 @@

      Spark Summit agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-east-2015-videos-posted.html b/site/news/spark-summit-east-2015-videos-posted.html index 72a7e3ff203..20bc53f11c4 100644 --- a/site/news/spark-summit-east-2015-videos-posted.html +++ b/site/news/spark-summit-east-2015-videos-posted.html @@ -141,6 +141,9 @@

      Spark Summit East 2015 Videos Posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-east-2016-cfp-closing.html b/site/news/spark-summit-east-2016-cfp-closing.html index 2661200f647..8b95e1a5a47 100644 --- a/site/news/spark-summit-east-2016-cfp-closing.html +++ b/site/news/spark-summit-east-2016-cfp-closing.html @@ -139,6 +139,9 @@

      CFP for Spark Summit East 2016 is closing soon!

      Latest News

      Archive

      diff --git a/site/news/spark-summit-east-2017-agenda-posted.html b/site/news/spark-summit-east-2017-agenda-posted.html index fe7d245d0a8..38c754b473b 100644 --- a/site/news/spark-summit-east-2017-agenda-posted.html +++ b/site/news/spark-summit-east-2017-agenda-posted.html @@ -139,6 +139,9 @@

      Spark Summit East (Feb 7-9th, 2017, Boston) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-east-agenda-posted-2015.html b/site/news/spark-summit-east-agenda-posted-2015.html index 30d1b9a3d16..3363474e449 100644 --- a/site/news/spark-summit-east-agenda-posted-2015.html +++ b/site/news/spark-summit-east-agenda-posted-2015.html @@ -141,6 +141,9 @@

      Spark Summit East agenda posted, CFP open for West

      Latest News

      Archive

      diff --git a/site/news/spark-summit-east-agenda-posted-2016.html b/site/news/spark-summit-east-agenda-posted-2016.html index 074fcde19a6..05504b24a32 100644 --- a/site/news/spark-summit-east-agenda-posted-2016.html +++ b/site/news/spark-summit-east-agenda-posted-2016.html @@ -139,6 +139,9 @@

      Spark Summit East (Feb 16, 2016, New York) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-eu-2017-agenda-posted.html b/site/news/spark-summit-eu-2017-agenda-posted.html index 7b500856a56..cdeeb759504 100644 --- a/site/news/spark-summit-eu-2017-agenda-posted.html +++ b/site/news/spark-summit-eu-2017-agenda-posted.html @@ -140,6 +140,9 @@

      Spark Summit Europe (October 24-26th, 2017, Dublin, Ireland) agenda postedLatest News

      Archive

      diff --git a/site/news/spark-summit-europe-agenda-posted.html b/site/news/spark-summit-europe-agenda-posted.html index 5e396404a46..0a053314cdb 100644 --- a/site/news/spark-summit-europe-agenda-posted.html +++ b/site/news/spark-summit-europe-agenda-posted.html @@ -139,6 +139,9 @@

      Spark Summit Europe agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-europe.html b/site/news/spark-summit-europe.html index f34d980bd4f..e16b83d0264 100644 --- a/site/news/spark-summit-europe.html +++ b/site/news/spark-summit-europe.html @@ -139,6 +139,9 @@

      Announcing Spark Summit Europe

      Latest News

      Archive

      diff --git a/site/news/spark-summit-june-2016-agenda-posted.html b/site/news/spark-summit-june-2016-agenda-posted.html index 5081aba9a07..7cf216c90d0 100644 --- a/site/news/spark-summit-june-2016-agenda-posted.html +++ b/site/news/spark-summit-june-2016-agenda-posted.html @@ -139,6 +139,9 @@

      Spark Summit (June 6, 2016, San Francisco) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-june-2017-agenda-posted.html b/site/news/spark-summit-june-2017-agenda-posted.html index 095e596c316..469e525ad9e 100644 --- a/site/news/spark-summit-june-2017-agenda-posted.html +++ b/site/news/spark-summit-june-2017-agenda-posted.html @@ -139,6 +139,9 @@

      Spark Summit (June 5-7th, 2017, San Francisco) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-june-2018-agenda-posted.html b/site/news/spark-summit-june-2018-agenda-posted.html index f22b6022c11..2334368fbb0 100644 --- a/site/news/spark-summit-june-2018-agenda-posted.html +++ b/site/news/spark-summit-june-2018-agenda-posted.html @@ -139,6 +139,9 @@

      Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-summit-oct-2018-agenda-posted.html b/site/news/spark-summit-oct-2018-agenda-posted.html index a5b969b8791..1c25bac8eb1 100644 --- a/site/news/spark-summit-oct-2018-agenda-posted.html +++ b/site/news/spark-summit-oct-2018-agenda-posted.html @@ -139,6 +139,9 @@

      Spark+AI Summit (October 2-4th, 2018, London) agenda posted

      Latest News

      Archive

      diff --git a/site/news/spark-tips-from-quantifind.html b/site/news/spark-tips-from-quantifind.html index 1169d6f2f64..5119fa4cf57 100644 --- a/site/news/spark-tips-from-quantifind.html +++ b/site/news/spark-tips-from-quantifind.html @@ -144,6 +144,9 @@

      Spark tips from Quantifind

      Latest News

      Archive

      diff --git a/site/news/spark-user-survey-and-powered-by-page.html b/site/news/spark-user-survey-and-powered-by-page.html index e38d1179a44..e94386307d4 100644 --- a/site/news/spark-user-survey-and-powered-by-page.html +++ b/site/news/spark-user-survey-and-powered-by-page.html @@ -143,6 +143,9 @@

      Spark user survey and "Powered By" page

      Latest News

      Archive

      diff --git a/site/news/spark-version-0-6-0-released.html b/site/news/spark-version-0-6-0-released.html index 3963fd37869..84976866b1a 100644 --- a/site/news/spark-version-0-6-0-released.html +++ b/site/news/spark-version-0-6-0-released.html @@ -139,6 +139,9 @@

      Spark version 0.6.0 released

      Latest News

      Archive

      diff --git a/site/news/spark-wins-cloudsort-100tb-benchmark.html b/site/news/spark-wins-cloudsort-100tb-benchmark.html index c492e4ee179..fe3bf34ef1e 100644 --- a/site/news/spark-wins-cloudsort-100tb-benchmark.html +++ b/site/news/spark-wins-cloudsort-100tb-benchmark.html @@ -146,6 +146,9 @@

      Spark wins CloudSort Benchmark as the most efficient engine

      Latest News

      Archive

      diff --git a/site/news/spark-wins-daytona-gray-sort-100tb-benchmark.html b/site/news/spark-wins-daytona-gray-sort-100tb-benchmark.html index ee544898272..a1d12ded9d6 100644 --- a/site/news/spark-wins-daytona-gray-sort-100tb-benchmark.html +++ b/site/news/spark-wins-daytona-gray-sort-100tb-benchmark.html @@ -145,6 +145,9 @@

      Spark wins Daytona Gray Sort 100TB Benchmark

      Latest News

      Archive

      diff --git a/site/news/strata-exercises-now-available-online.html b/site/news/strata-exercises-now-available-online.html index fff4952e8f7..a878e8d042a 100644 --- a/site/news/strata-exercises-now-available-online.html +++ b/site/news/strata-exercises-now-available-online.html @@ -139,6 +139,9 @@

      Strata exercises now available online

      Latest News

      Archive

      diff --git a/site/news/submit-talks-to-spark-summit-2014.html b/site/news/submit-talks-to-spark-summit-2014.html index 80df812b7cd..46367144f36 100644 --- a/site/news/submit-talks-to-spark-summit-2014.html +++ b/site/news/submit-talks-to-spark-summit-2014.html @@ -148,6 +148,9 @@

      Submissions and registration open for Spark Summit 2014

      Latest News

      Archive

      diff --git a/site/news/submit-talks-to-spark-summit-2016.html b/site/news/submit-talks-to-spark-summit-2016.html index d9c9428c4a4..413c8b1e988 100644 --- a/site/news/submit-talks-to-spark-summit-2016.html +++ b/site/news/submit-talks-to-spark-summit-2016.html @@ -140,6 +140,9 @@

      Submission is open for Spark Summit San Francisco

      Latest News

      Archive

      diff --git a/site/news/submit-talks-to-spark-summit-east-2016.html b/site/news/submit-talks-to-spark-summit-east-2016.html index 70996c0acc5..116bd3cb783 100644 --- a/site/news/submit-talks-to-spark-summit-east-2016.html +++ b/site/news/submit-talks-to-spark-summit-east-2016.html @@ -139,6 +139,9 @@

      Submission is open for Spark Summit East 2016

      Latest News

      Archive

      diff --git a/site/news/submit-talks-to-spark-summit-eu-2016.html b/site/news/submit-talks-to-spark-summit-eu-2016.html index f74841802ea..b869df05b6f 100644 --- a/site/news/submit-talks-to-spark-summit-eu-2016.html +++ b/site/news/submit-talks-to-spark-summit-eu-2016.html @@ -139,6 +139,9 @@

      Call for Presentations for Spark Summit EU is Open

      Latest News

      Archive

      diff --git a/site/news/two-weeks-to-spark-summit-2014.html b/site/news/two-weeks-to-spark-summit-2014.html index 914d5426739..775ac38129c 100644 --- a/site/news/two-weeks-to-spark-summit-2014.html +++ b/site/news/two-weeks-to-spark-summit-2014.html @@ -149,6 +149,9 @@

      Two weeks to Spark Summit 2014

      Latest News

      Archive

      diff --git a/site/news/video-from-first-spark-development-meetup.html b/site/news/video-from-first-spark-development-meetup.html index c69eeabffb7..c9f4629d2cd 100644 --- a/site/news/video-from-first-spark-development-meetup.html +++ b/site/news/video-from-first-spark-development-meetup.html @@ -139,6 +139,9 @@

      Video up from first Spark development meetup

      Latest News

      Archive

      diff --git a/site/powered-by.html b/site/powered-by.html index 46eb0b88895..473e6b3061f 100644 --- a/site/powered-by.html +++ b/site/powered-by.html @@ -490,6 +490,9 @@

      Companies and organizations

      Latest News

      Archive

      diff --git a/site/release-process.html b/site/release-process.html index 319600a8a43..8476716f094 100644 --- a/site/release-process.html +++ b/site/release-process.html @@ -541,6 +541,9 @@

      Create an announcement

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-3.html b/site/releases/spark-release-0-3.html index 4c954b9082d..992ff8305e9 100644 --- a/site/releases/spark-release-0-3.html +++ b/site/releases/spark-release-0-3.html @@ -190,6 +190,9 @@

      Other Features

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-5-0.html b/site/releases/spark-release-0-5-0.html index 9fb958f2980..f722eae5e73 100644 --- a/site/releases/spark-release-0-5-0.html +++ b/site/releases/spark-release-0-5-0.html @@ -162,6 +162,9 @@

      New Hadoop API Support

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-5-1.html b/site/releases/spark-release-0-5-1.html index 63107cfac22..14974be5cb7 100644 --- a/site/releases/spark-release-0-5-1.html +++ b/site/releases/spark-release-0-5-1.html @@ -172,6 +172,9 @@

      EC2 Improvements

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-5-2.html b/site/releases/spark-release-0-5-2.html index 16fe83edb0d..22aa408a115 100644 --- a/site/releases/spark-release-0-5-2.html +++ b/site/releases/spark-release-0-5-2.html @@ -141,6 +141,9 @@

      Spark Release 0.5.2

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-6-0.html b/site/releases/spark-release-0-6-0.html index fe591ddab60..ddb24bb659f 100644 --- a/site/releases/spark-release-0-6-0.html +++ b/site/releases/spark-release-0-6-0.html @@ -216,6 +216,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-6-1.html b/site/releases/spark-release-0-6-1.html index 5b5eab6a214..a18686bbc15 100644 --- a/site/releases/spark-release-0-6-1.html +++ b/site/releases/spark-release-0-6-1.html @@ -156,6 +156,9 @@

      Spark Release 0.6.1

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-6-2.html b/site/releases/spark-release-0-6-2.html index 0ae0424a2df..f9b8f7cf375 100644 --- a/site/releases/spark-release-0-6-2.html +++ b/site/releases/spark-release-0-6-2.html @@ -169,6 +169,9 @@

      Spark Release 0.6.2

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-7-0.html b/site/releases/spark-release-0-7-0.html index fb82d5ccc11..8798b3e05ff 100644 --- a/site/releases/spark-release-0-7-0.html +++ b/site/releases/spark-release-0-7-0.html @@ -238,6 +238,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-7-2.html b/site/releases/spark-release-0-7-2.html index c0c50f79e22..39ad2b2edc6 100644 --- a/site/releases/spark-release-0-7-2.html +++ b/site/releases/spark-release-0-7-2.html @@ -180,6 +180,9 @@

      Spark Release 0.7.2

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-7-3.html b/site/releases/spark-release-0-7-3.html index 80f39acea40..ae9ce232a69 100644 --- a/site/releases/spark-release-0-7-3.html +++ b/site/releases/spark-release-0-7-3.html @@ -174,6 +174,9 @@

      Spark Release 0.7.3

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-8-0.html b/site/releases/spark-release-0-8-0.html index 97c63f96140..ae946537eb9 100644 --- a/site/releases/spark-release-0-8-0.html +++ b/site/releases/spark-release-0-8-0.html @@ -269,6 +269,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-8-1.html b/site/releases/spark-release-0-8-1.html index eb1ad09855a..8b9311764fa 100644 --- a/site/releases/spark-release-0-8-1.html +++ b/site/releases/spark-release-0-8-1.html @@ -233,6 +233,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-9-0.html b/site/releases/spark-release-0-9-0.html index 75a626de607..c62bb47e840 100644 --- a/site/releases/spark-release-0-9-0.html +++ b/site/releases/spark-release-0-9-0.html @@ -327,6 +327,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-9-1.html b/site/releases/spark-release-0-9-1.html index e5ce6a99e04..6dbc3c88393 100644 --- a/site/releases/spark-release-0-9-1.html +++ b/site/releases/spark-release-0-9-1.html @@ -245,6 +245,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-0-9-2.html b/site/releases/spark-release-0-9-2.html index fb92991c766..cf42a7928cf 100644 --- a/site/releases/spark-release-0-9-2.html +++ b/site/releases/spark-release-0-9-2.html @@ -218,6 +218,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-0-0.html b/site/releases/spark-release-1-0-0.html index 36918a1996e..413c6d0ab2e 100644 --- a/site/releases/spark-release-1-0-0.html +++ b/site/releases/spark-release-1-0-0.html @@ -311,6 +311,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-0-1.html b/site/releases/spark-release-1-0-1.html index 891b17c861d..27a332b43bc 100644 --- a/site/releases/spark-release-1-0-1.html +++ b/site/releases/spark-release-1-0-1.html @@ -269,6 +269,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-0-2.html b/site/releases/spark-release-1-0-2.html index 5ef0c7d6934..f68a5fea8ae 100644 --- a/site/releases/spark-release-1-0-2.html +++ b/site/releases/spark-release-1-0-2.html @@ -226,6 +226,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-1-0.html b/site/releases/spark-release-1-1-0.html index f2bf65c1150..0d432e1fa98 100644 --- a/site/releases/spark-release-1-1-0.html +++ b/site/releases/spark-release-1-1-0.html @@ -362,6 +362,9 @@

      Contributors:

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-1-1.html b/site/releases/spark-release-1-1-1.html index cca4e9bbcab..5b5bdeaf6d8 100644 --- a/site/releases/spark-release-1-1-1.html +++ b/site/releases/spark-release-1-1-1.html @@ -248,6 +248,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-2-0.html b/site/releases/spark-release-1-2-0.html index 074a9a4f889..665375b0fac 100644 --- a/site/releases/spark-release-1-2-0.html +++ b/site/releases/spark-release-1-2-0.html @@ -376,6 +376,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-2-1.html b/site/releases/spark-release-1-2-1.html index 18d811405b9..8b29777933e 100644 --- a/site/releases/spark-release-1-2-1.html +++ b/site/releases/spark-release-1-2-1.html @@ -248,6 +248,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-2-2.html b/site/releases/spark-release-1-2-2.html index fa6413f5289..270c09b40a5 100644 --- a/site/releases/spark-release-1-2-2.html +++ b/site/releases/spark-release-1-2-2.html @@ -208,6 +208,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-3-0.html b/site/releases/spark-release-1-3-0.html index 4f61aecd458..0452586242b 100644 --- a/site/releases/spark-release-1-3-0.html +++ b/site/releases/spark-release-1-3-0.html @@ -353,6 +353,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-3-1.html b/site/releases/spark-release-1-3-1.html index 90d9a0f73f7..efbdb21027c 100644 --- a/site/releases/spark-release-1-3-1.html +++ b/site/releases/spark-release-1-3-1.html @@ -240,6 +240,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-4-0.html b/site/releases/spark-release-1-4-0.html index 014d9bd1240..09c65ad6d73 100644 --- a/site/releases/spark-release-1-4-0.html +++ b/site/releases/spark-release-1-4-0.html @@ -467,6 +467,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-4-1.html b/site/releases/spark-release-1-4-1.html index 35f0427abfb..f90c129e347 100644 --- a/site/releases/spark-release-1-4-1.html +++ b/site/releases/spark-release-1-4-1.html @@ -272,6 +272,9 @@

      Contributors

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-5-0.html b/site/releases/spark-release-1-5-0.html index 4e8c40d9f9c..caa95b440e2 100644 --- a/site/releases/spark-release-1-5-0.html +++ b/site/releases/spark-release-1-5-0.html @@ -405,6 +405,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-5-1.html b/site/releases/spark-release-1-5-1.html index 24799985099..28424b101d8 100644 --- a/site/releases/spark-release-1-5-1.html +++ b/site/releases/spark-release-1-5-1.html @@ -144,6 +144,9 @@

      Spark Release 1.5.1

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-5-2.html b/site/releases/spark-release-1-5-2.html index 070446af75d..cf477710fec 100644 --- a/site/releases/spark-release-1-5-2.html +++ b/site/releases/spark-release-1-5-2.html @@ -143,6 +143,9 @@

      Spark Release 1.5.2

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-6-0.html b/site/releases/spark-release-1-6-0.html index 5fe8d623965..2e54f0e204e 100644 --- a/site/releases/spark-release-1-6-0.html +++ b/site/releases/spark-release-1-6-0.html @@ -281,6 +281,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-6-1.html b/site/releases/spark-release-1-6-1.html index 9f64cdd38bb..c6d91e8241d 100644 --- a/site/releases/spark-release-1-6-1.html +++ b/site/releases/spark-release-1-6-1.html @@ -143,6 +143,9 @@

      Spark Release 1.6.1

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-6-2.html b/site/releases/spark-release-1-6-2.html index b1bb440dc5c..f73d743ffa3 100644 --- a/site/releases/spark-release-1-6-2.html +++ b/site/releases/spark-release-1-6-2.html @@ -143,6 +143,9 @@

      Spark Release 1.6.2

      Latest News

      Archive

      diff --git a/site/releases/spark-release-1-6-3.html b/site/releases/spark-release-1-6-3.html index bc7b41bf2d6..217961a2c48 100644 --- a/site/releases/spark-release-1-6-3.html +++ b/site/releases/spark-release-1-6-3.html @@ -143,6 +143,9 @@

      Spark Release 1.6.3

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-0-0.html b/site/releases/spark-release-2-0-0.html index 365302939d9..8274befbb95 100644 --- a/site/releases/spark-release-2-0-0.html +++ b/site/releases/spark-release-2-0-0.html @@ -377,6 +377,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-0-1.html b/site/releases/spark-release-2-0-1.html index e814832e769..d55bfb60f02 100644 --- a/site/releases/spark-release-2-0-1.html +++ b/site/releases/spark-release-2-0-1.html @@ -143,6 +143,9 @@

      Spark Release 2.0.1

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-0-2.html b/site/releases/spark-release-2-0-2.html index bf650c1c933..40554d676cb 100644 --- a/site/releases/spark-release-2-0-2.html +++ b/site/releases/spark-release-2-0-2.html @@ -145,6 +145,9 @@

      Spark Release 2.0.2

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-1-0.html b/site/releases/spark-release-2-1-0.html index c6e349bf3dc..b7754bd194f 100644 --- a/site/releases/spark-release-2-1-0.html +++ b/site/releases/spark-release-2-1-0.html @@ -289,6 +289,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-1-1.html b/site/releases/spark-release-2-1-1.html index 1758de99b41..35734e23f90 100644 --- a/site/releases/spark-release-2-1-1.html +++ b/site/releases/spark-release-2-1-1.html @@ -143,6 +143,9 @@

      Spark Release 2.1.1

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-1-2.html b/site/releases/spark-release-2-1-2.html index 50aa541fca5..ee778b6f05e 100644 --- a/site/releases/spark-release-2-1-2.html +++ b/site/releases/spark-release-2-1-2.html @@ -143,6 +143,9 @@

      Spark Release 2.1.2

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-1-3.html b/site/releases/spark-release-2-1-3.html index 62d0ede4a0d..bf55793ec28 100644 --- a/site/releases/spark-release-2-1-3.html +++ b/site/releases/spark-release-2-1-3.html @@ -143,6 +143,9 @@

      Spark Release 2.1.3

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-2-0.html b/site/releases/spark-release-2-2-0.html index 91b5d69a79b..004c30e6496 100644 --- a/site/releases/spark-release-2-2-0.html +++ b/site/releases/spark-release-2-2-0.html @@ -355,6 +355,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-2-1.html b/site/releases/spark-release-2-2-1.html index 27d5a3c8629..00b3a7d4539 100644 --- a/site/releases/spark-release-2-2-1.html +++ b/site/releases/spark-release-2-2-1.html @@ -153,6 +153,9 @@

      Changes of behavior

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-2-2.html b/site/releases/spark-release-2-2-2.html index 6a6cd1ece88..f95517e2f49 100644 --- a/site/releases/spark-release-2-2-2.html +++ b/site/releases/spark-release-2-2-2.html @@ -145,6 +145,9 @@

      Changes of behavior

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-2-3.html b/site/releases/spark-release-2-2-3.html index 1466d44b168..ac7c3a6f15d 100644 --- a/site/releases/spark-release-2-2-3.html +++ b/site/releases/spark-release-2-2-3.html @@ -143,6 +143,9 @@

      Spark Release 2.2.3

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-3-0.html b/site/releases/spark-release-2-3-0.html index 95a725f8486..ead2de52cb1 100644 --- a/site/releases/spark-release-2-3-0.html +++ b/site/releases/spark-release-2-3-0.html @@ -365,6 +365,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-3-1.html b/site/releases/spark-release-2-3-1.html index 54885dfb978..48587b064b2 100644 --- a/site/releases/spark-release-2-3-1.html +++ b/site/releases/spark-release-2-3-1.html @@ -153,6 +153,9 @@

      Changes of behavior

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-3-2.html b/site/releases/spark-release-2-3-2.html index ec9a679c391..f17e09fd71d 100644 --- a/site/releases/spark-release-2-3-2.html +++ b/site/releases/spark-release-2-3-2.html @@ -153,6 +153,9 @@

      Know issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-3-3.html b/site/releases/spark-release-2-3-3.html index 5598cb6604c..2c709a9ea1d 100644 --- a/site/releases/spark-release-2-3-3.html +++ b/site/releases/spark-release-2-3-3.html @@ -143,6 +143,9 @@

      Spark Release 2.3.3

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-3-4.html b/site/releases/spark-release-2-3-4.html index e715040adb7..9c4596e426f 100644 --- a/site/releases/spark-release-2-3-4.html +++ b/site/releases/spark-release-2-3-4.html @@ -143,6 +143,9 @@

      Spark Release 2.3.4

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-0.html b/site/releases/spark-release-2-4-0.html index 66b2d2f23cd..dd090507edf 100644 --- a/site/releases/spark-release-2-4-0.html +++ b/site/releases/spark-release-2-4-0.html @@ -371,6 +371,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-1.html b/site/releases/spark-release-2-4-1.html index 2ce920089e6..5baf58282f1 100644 --- a/site/releases/spark-release-2-4-1.html +++ b/site/releases/spark-release-2-4-1.html @@ -181,6 +181,9 @@

      Known issue

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-2.html b/site/releases/spark-release-2-4-2.html index c124b88e68c..6613714ec0f 100644 --- a/site/releases/spark-release-2-4-2.html +++ b/site/releases/spark-release-2-4-2.html @@ -153,6 +153,9 @@

      Notable changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-3.html b/site/releases/spark-release-2-4-3.html index 903f6153577..f93fa291577 100644 --- a/site/releases/spark-release-2-4-3.html +++ b/site/releases/spark-release-2-4-3.html @@ -151,6 +151,9 @@

      Notable changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-4.html b/site/releases/spark-release-2-4-4.html index 039f07f916d..16e55366ee2 100644 --- a/site/releases/spark-release-2-4-4.html +++ b/site/releases/spark-release-2-4-4.html @@ -157,6 +157,9 @@

      Notable changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-5.html b/site/releases/spark-release-2-4-5.html index e599e1479cd..bfb2cd8bf19 100644 --- a/site/releases/spark-release-2-4-5.html +++ b/site/releases/spark-release-2-4-5.html @@ -171,6 +171,9 @@

      Known issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-6.html b/site/releases/spark-release-2-4-6.html index 80aa339a502..84f4df83756 100644 --- a/site/releases/spark-release-2-4-6.html +++ b/site/releases/spark-release-2-4-6.html @@ -179,6 +179,9 @@

      Known issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-7.html b/site/releases/spark-release-2-4-7.html index f6d2b9047df..dc0ebc6a349 100644 --- a/site/releases/spark-release-2-4-7.html +++ b/site/releases/spark-release-2-4-7.html @@ -221,6 +221,9 @@

      Known issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-2-4-8.html b/site/releases/spark-release-2-4-8.html index a3f9ef3244d..d83c75e0ea0 100644 --- a/site/releases/spark-release-2-4-8.html +++ b/site/releases/spark-release-2-4-8.html @@ -222,6 +222,9 @@

      Known issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-0-0.html b/site/releases/spark-release-3-0-0.html index 75559771527..9c6e90f54f4 100644 --- a/site/releases/spark-release-3-0-0.html +++ b/site/releases/spark-release-3-0-0.html @@ -531,6 +531,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-0-1.html b/site/releases/spark-release-3-0-1.html index 039f306ec94..cdbadc2ae54 100644 --- a/site/releases/spark-release-3-0-1.html +++ b/site/releases/spark-release-3-0-1.html @@ -180,6 +180,9 @@

      Known issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-0-2.html b/site/releases/spark-release-3-0-2.html index e5e2ad6f097..b5c989703ad 100644 --- a/site/releases/spark-release-3-0-2.html +++ b/site/releases/spark-release-3-0-2.html @@ -190,6 +190,9 @@

      Known issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-0-3.html b/site/releases/spark-release-3-0-3.html index 221440bc821..8b2c690e365 100644 --- a/site/releases/spark-release-3-0-3.html +++ b/site/releases/spark-release-3-0-3.html @@ -207,6 +207,9 @@

      Known issues

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-1-1.html b/site/releases/spark-release-3-1-1.html index 9b5c679fec8..95af3d8a6ed 100644 --- a/site/releases/spark-release-3-1-1.html +++ b/site/releases/spark-release-3-1-1.html @@ -568,6 +568,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-1-2.html b/site/releases/spark-release-3-1-2.html index b8c462d0154..7991fc5a34c 100644 --- a/site/releases/spark-release-3-1-2.html +++ b/site/releases/spark-release-3-1-2.html @@ -255,6 +255,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-1-3.html b/site/releases/spark-release-3-1-3.html index 3dc0881c861..b490933ae08 100644 --- a/site/releases/spark-release-3-1-3.html +++ b/site/releases/spark-release-3-1-3.html @@ -169,6 +169,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-2-0.html b/site/releases/spark-release-3-2-0.html index adbe12bbbd4..b631ae34e17 100644 --- a/site/releases/spark-release-3-2-0.html +++ b/site/releases/spark-release-3-2-0.html @@ -541,6 +541,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-2-1.html b/site/releases/spark-release-3-2-1.html index 61803e4466f..9c5011abd66 100644 --- a/site/releases/spark-release-3-2-1.html +++ b/site/releases/spark-release-3-2-1.html @@ -174,6 +174,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-2-2.html b/site/releases/spark-release-3-2-2.html index c6479789647..2899c708c96 100644 --- a/site/releases/spark-release-3-2-2.html +++ b/site/releases/spark-release-3-2-2.html @@ -249,6 +249,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-2-3.html b/site/releases/spark-release-3-2-3.html index a686e7767bd..2ef0e529f1e 100644 --- a/site/releases/spark-release-3-2-3.html +++ b/site/releases/spark-release-3-2-3.html @@ -222,6 +222,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-2-4.html b/site/releases/spark-release-3-2-4.html index 06b1d7b28c3..b5270ea83d2 100644 --- a/site/releases/spark-release-3-2-4.html +++ b/site/releases/spark-release-3-2-4.html @@ -184,6 +184,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-3-0.html b/site/releases/spark-release-3-3-0.html index 73c1fe58859..b21cd529c12 100644 --- a/site/releases/spark-release-3-3-0.html +++ b/site/releases/spark-release-3-3-0.html @@ -721,6 +721,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-3-1.html b/site/releases/spark-release-3-3-1.html index aa080cfb08e..a1486b99f42 100644 --- a/site/releases/spark-release-3-3-1.html +++ b/site/releases/spark-release-3-3-1.html @@ -231,6 +231,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-3-2.html b/site/releases/spark-release-3-3-2.html index 860813ba3a1..558c99b8a46 100644 --- a/site/releases/spark-release-3-3-2.html +++ b/site/releases/spark-release-3-3-2.html @@ -308,6 +308,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-3-3.html b/site/releases/spark-release-3-3-3.html index 06604dacc2c..0675e15a0e3 100644 --- a/site/releases/spark-release-3-3-3.html +++ b/site/releases/spark-release-3-3-3.html @@ -163,6 +163,9 @@

      Notable changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-4-0.html b/site/releases/spark-release-3-4-0.html index f68b41ca4c7..60ecb8dc8ed 100644 --- a/site/releases/spark-release-3-4-0.html +++ b/site/releases/spark-release-3-4-0.html @@ -549,6 +549,9 @@

      Credits

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-4-1.html b/site/releases/spark-release-3-4-1.html index 443dd8cb659..00c46540824 100644 --- a/site/releases/spark-release-3-4-1.html +++ b/site/releases/spark-release-3-4-1.html @@ -236,6 +236,9 @@

      Dependency Changes

      Latest News

      Archive

      diff --git a/site/releases/spark-release-3-5-0.html b/site/releases/spark-release-3-5-0.html new file mode 100644 index 00000000000..129ee4712b3 --- /dev/null +++ b/site/releases/spark-release-3-5-0.html @@ -0,0 +1,579 @@ + + + + + + + + + Spark Release 3.5.0 | Apache Spark + + + + + + + + + + + + + + + + + + + +
      +
      +
      +

      Spark Release 3.5.0

      + + +

      Apache Spark 3.5.0 is the sixth release in the 3.x series. With significant contributions from the open-source community, this release addressed over 1,300 Jira tickets.

      + +

      This release introduces more scenarios with general availability for Spark Connect, like Scala and Go client, distributed training and inference support, and enhancement of compatibility for Structured streaming; introduces new PySpark and SQL functionality such as like SQL IDENTIFIER clause, named argument support for SQL function calls, SQL function support for HyperLogLog approximate aggregations, and Python user-defined table functions; simplifies distributed training with DeepSpeed; introduces watermark propagation among operators, introduces dropDuplicatesWithinWatermark operations in Structured Streaming.

      + +

      To download Apache Spark 3.5.0, please visit the downloads page. For detailed changes, you can consult JIRA. We have also curated a list of high-level changes here, grouped by major modules.

      + + + +

      Highlights

      + +
        +
      • Scala and Go client support in Spark Connect SPARK-42554 SPARK-43351
        • +
      • PyTorch-based distributed ML Support for Spark Connect SPARK-42471
        • +
      • Structured Streaming support for Spark Connect in Python and Scala SPARK-42938
        • +
      • Pandas API support for the Python Spark Connect Client SPARK-42497
        • +
      • Introduce Arrow Python UDFs SPARK-40307
        • +
      • Support Python user-defined table functions SPARK-43798
        • +
      • Migrate PySpark errors onto error classes SPARK-42986
        • +
      • PySpark Test Framework SPARK-44042
        • +
      • Add support for Datasketches HllSketch SPARK-16484
        • +
      • Built-in SQL Function Improvement SPARK-41231
        • +
      • IDENTIFIER clause SPARK-43205
        • +
      • Add SQL functions into Scala, Python and R API SPARK-43907
        • +
      • Add named argument support for SQL functions SPARK-43922
        • +
      • Avoid unnecessary task rerun on decommissioned executor lost if shuffle data migrated SPARK-41469
        • +
      • Distributed ML <> spark connect SPARK-42471
        • +
      • DeepSpeed Distributor SPARK-44264
        • +
      • Implement changelog checkpointing for RocksDB state store SPARK-43421
        • +
      • Introduce watermark propagation among operators SPARK-42376
        • +
      • Introduce dropDuplicatesWithinWatermark SPARK-42931
        • +
      • RocksDB state store provider memory management enhancements SPARK-43311
        • +
      + +

      Spark Connect

      + +
        +
      • Refactoring of the sql module into sql and sql-api to produce a minimum set of dependencies that can be shared between the Scala Spark Connect client and Spark and avoids pulling all of the Spark transitive dependencies. SPARK-44273
        • +
      • Introducing the Scala client for Spark Connect SPARK-42554
        • +
      • Pandas API support for the Python Spark Connect Client SPARK-42497
        • +
      • PyTorch-based distributed ML Support for Spark Connect SPARK-42471
        • +
      • Structured Streaming support for Spark Connect in Python and Scala SPARK-42938
        • +
      • Initial version of the Go client SPARK-43351
        • +
      • Lot’s of compatibility improvements between Spark native and the Spark Connect clients across Python and Scala
        • +
      • Improved debugability and request handling for client applications (asynchronous processing, retries, long-lived queries)
        • +
      + +

      Spark SQL

      + +

      Features

      + +
        +
      • Add metadata column file block start and length SPARK-42423
        • +
      • Support positional parameters in Scala/Java sql() SPARK-44066
        • +
      • Add named parameter support in parser for function calls SPARK-43922
        • +
      • Support SELECT DEFAULT with ORDER BY, LIMIT, OFFSET for INSERT source relation SPARK-43071
        • +
      • Add SQL grammar for PARTITION BY and ORDER BY clause after TABLE arguments for TVF calls SPARK-44503
        • +
      • Include column default values in DESCRIBE and SHOW CREATE TABLE output SPARK-42123
        • +
      • Add optional pattern for Catalog.listCatalogs SPARK-43792
        • +
      • Add optional pattern for Catalog.listDatabases SPARK-43881
        • +
      • Callback when ready for execution SPARK-44145
        • +
      • Support Insert By Name statement SPARK-42750
        • +
      • Add call_function for Scala API SPARK-44131
        • +
      • Stable derived column aliases SPARK-40822
        • +
      • Support general constant expressions as CREATE/REPLACE TABLE OPTIONS values SPARK-43529
        • +
      • Support subqueries with correlation through INTERSECT/EXCEPT SPARK-36124
        • +
      • IDENTIFIER clause SPARK-43205
        • +
      • ANSI MODE: Conv should return an error if the internal conversion overflows SPARK-42427
        • +
      + +

      Functions

      + +
        +
      • Add support for Datasketches HllSketch SPARK-16484
        • +
      • Support the CBC mode by aes_encrypt()/aes_decrypt() SPARK-43038
        • +
      • Support TABLE argument parser rule for TableValuedFunction SPARK-44200
        • +
      • Implement bitmap functions SPARK-44154
        • +
      • Add the try_aes_decrypt() function SPARK-42701
        • +
      • array_insert should fail with 0 index SPARK-43011
        • +
      • Add to_varchar alias for to_char SPARK-43815
        • +
      • High-order function: array_compact implementation SPARK-41235
        • +
      • Add analyzer support of named arguments for built-in functions SPARK-44059
        • +
      • Add NULLs for INSERTs with user-specified lists of fewer columns than the target table SPARK-42521
        • +
      • Adds support for aes_encrypt IVs and AAD SPARK-43290
        • +
      • DECODE function returns wrong results when passed NULL SPARK-41668
        • +
      • Support udf ‘luhn_check’ SPARK-42191
        • +
      • Support implicit lateral column alias resolution on Aggregate SPARK-41631
        • +
      • Support implicit lateral column alias in queries with Window SPARK-42217
        • +
      • Add 3-args function aliases DATE_ADD and DATE_DIFF SPARK-43492
        • +
      + +

      Data Sources

      + +
        +
      • Char/Varchar Support for JDBC Catalog SPARK-42904
        • +
      • Support Get SQL Keywords Dynamically Thru JDBC API and TVF SPARK-43119
        • +
      • DataSource V2: Handle MERGE commands for delta-based sources SPARK-43885
        • +
      • DataSource V2: Handle MERGE commands for group-based sources SPARK-43963
        • +
      • DataSource V2: Handle UPDATE commands for group-based sources SPARK-43975
        • +
      • DataSource V2: Allow representing updates as deletes and inserts SPARK-43775
        • +
      • Allow jdbc dialects to override the query used to create a table SPARK-41516
        • +
      • SPJ: Support partially clustered distribution SPARK-42038
        • +
      • DSv2 allows CTAS/RTAS to reserve schema nullability SPARK-43390
        • +
      • Add spark.sql.files.maxPartitionNum SPARK-44021
        • +
      • Handle UPDATE commands for delta-based sources SPARK-43324
        • +
      • Allow V2 writes to indicate advisory shuffle partition size SPARK-42779
        • +
      • Support lz4raw compression codec for Parquet SPARK-43273
        • +
      • Avro: writing complex unions SPARK-25050
        • +
      • Speed up Timestamp type inference with user-provided format in JSON/CSV data source SPARK-39280
        • +
      • Avro to Support custom decimal type backed by Long SPARK-43901
        • +
      • Avoid shuffle in Storage-Partitioned Join when partition keys mismatch, but join expressions are compatible SPARK-41413
        • +
      • Change binary to unsupported dataType in CSV format SPARK-42237
        • +
      • Allow Avro to convert union type to SQL with field name stable with type SPARK-43333
        • +
      • Speed up Timestamp type inference with legacy format in JSON/CSV data source SPARK-39281
        • +
      + +

      Query Optimization

      + +
        +
      • Subexpression elimination support shortcut expression SPARK-42815
        • +
      • Improve join stats estimation if one side can keep uniqueness SPARK-39851
        • +
      • Introduce the group limit of Window for rank-based filter to optimize top-k computation SPARK-37099
        • +
      • Fix behavior of null IN (empty list) in optimization rules SPARK-44431
        • +
      • Infer and push down window limit through window if partitionSpec is empty SPARK-41171
        • +
      • Remove the outer join if they are all distinct aggregate functions SPARK-42583
        • +
      • Collapse two adjacent windows with the same partition/order in subquery SPARK-42525
        • +
      • Push down limit through Python UDFs SPARK-42115
        • +
      • Optimize the order of filtering predicates SPARK-40045
        • +
      + +

      Code Generation and Query Execution

      + +
        +
      • Runtime filter should supports multi level shuffle join side as filter creation side SPARK-41674
        • +
      • Codegen Support for HiveSimpleUDF SPARK-42052
        • +
      • Codegen Support for HiveGenericUDF SPARK-42051
        • +
      • Codegen Support for build side outer shuffled hash join SPARK-44060
        • +
      • Implement code generation for to_csv function (StructsToCsv) SPARK-42169
        • +
      • Make AQE support InMemoryTableScanExec SPARK-42101
        • +
      • Support left outer join build left or right outer join build right in shuffled hash join SPARK-36612
        • +
      • Respect RequiresDistributionAndOrdering in CTAS/RTAS SPARK-43088
        • +
      • Coalesce buckets in join applied on broadcast join stream side SPARK-43107
        • +
      • Set nullable correctly on coalesced join key in full outer USING join SPARK-44251
        • +
      • Fix IN subquery ListQuery nullability SPARK-43413
        • +
      + +

      Other Notable Changes

      + +
        +
      • Set nullable correctly for keys in USING joins SPARK-43718
        • +
      • Fix COUNT(*) is null bug in correlated scalar subquery SPARK-43156
        • +
      • Dataframe.joinWith outer-join should return a null value for unmatched row SPARK-37829
        • +
      • Automatically rename conflicting metadata columns SPARK-42683
        • +
      • Document the Spark SQL error classes in user-facing documentation SPARK-42706
        • +
      + +

      PySpark

      + +

      Features

      + +
        +
      • Support positional parameters in Python sql() SPARK-44140
        • +
      • Support parameterized SQL by sql() SPARK-41666
        • +
      • Support Python user-defined table functions SPARK-43797
        • +
      • Support to set Python executable for UDF and pandas function APIs in workers during runtime SPARK-43574
        • +
      • Add DataFrame.offset to PySpark SPARK-43213
        • +
      • Implement dir() in pyspark.sql.dataframe.DataFrame to include columns SPARK-43270
        • +
      • Add option to use large variable width vectors for arrow UDF operations SPARK-39979
        • +
      • Make mapInPandas / mapInArrow support barrier mode execution SPARK-42896
        • +
      • Add JobTag APIs to PySpark SparkContext SPARK-44194
        • +
      • Support for Python UDTF to analyze in Python SPARK-44380
        • +
      • Expose TimestampNTZType in pyspark.sql.types SPARK-43759
        • +
      • Support nested timestamp type SPARK-43545
        • +
      • Support UserDefinedType in createDataFrame from pandas DataFrame and toPandas [SPARK-43817]SPARK-43702
        • +
      • Add descriptor binary option to Pyspark Protobuf API SPARK-43799
        • +
      • Accept generics tuple as typing hints of Pandas UDF SPARK-43886
        • +
      • Add array_prepend function SPARK-41233
        • +
      • Add assertDataFrameEqual util function SPARK-44061
        • +
      • Support arrow-optimized Python UDTFs SPARK-43964
        • +
      • Allow custom precision for fp approx equality SPARK-44217
        • +
      • Make assertSchemaEqual API public SPARK-44216
        • +
      • Support fill_value for ps.Series SPARK-42094
        • +
      • Support struct type in createDataFrame from pandas DataFrame SPARK-43473
        • +
      + +

      Other Notable Changes

      + +
        +
      • + + + + + + + +
        Add autocomplete support for df[] in pyspark.sql.dataframe.DataFrame [SPARK-43892]
        +
        • +
      • Deprecate & remove the APIs that will be removed in pandas 2.0 [SPARK-42593]
        • +
      • Make Python the first tab for code examples - Spark SQL, DataFrames and Datasets Guide SPARK-42493
        • +
      • Updating remaining Spark documentation code examples to show Python by default SPARK-42642
        • +
      • Use deduplicated field names when creating Arrow RecordBatch [SPARK-41971]
        • +
      • Support duplicated field names in createDataFrame with pandas DataFrame [SPARK-43528]
        • +
      • Allow columns parameter when creating DataFrame with Series [SPARK-42194]
        • +
      + +

      Core

      + +
        +
      • Schedule mergeFinalize when push merge shuffleMapStage retry but no running tasks SPARK-40082
        • +
      • Introduce PartitionEvaluator for SQL operator execution SPARK-43061
        • +
      • Allow ShuffleDriverComponent to declare if shuffle data is reliably stored SPARK-42689
        • +
      • Add max attempts limitation for stages to avoid potential infinite retry SPARK-42577
        • +
      • Support log level configuration with static Spark conf SPARK-43782
        • +
      • Optimize PercentileHeap SPARK-42528
        • +
      • Add reason argument to TaskScheduler.cancelTasks SPARK-42602
        • +
      • Avoid unnecessary task rerun on decommissioned executor lost if shuffle data migrated SPARK-41469
        • +
      • Fixing accumulator undercount in the case of the retry task with rdd cache SPARK-41497
        • +
      • Use RocksDB for spark.history.store.hybridStore.diskBackend by default SPARK-42277
        • +
      • Support spark.kubernetes.setSubmitTimeInDriver SPARK-43014
        • +
      • NonFateSharingCache wrapper for Guava Cache SPARK-43300
        • +
      • Improve the performance of MapOutputTracker.updateMapOutput SPARK-43043
        • +
      • Allowing apps to control whether their metadata gets saved in the db by the External Shuffle Service SPARK-43179
        • +
      • Port executor failure tracker from Spark on YARN to K8s SPARK-41210
        • +
      • Parameterize the max number of attempts for driver props fetcher in KubernetesExecutorBackend SPARK-42764
        • +
      • Add SPARK_DRIVER_POD_IP env variable to executor pods SPARK-42769
        • +
      • Mounts the hadoop config map on the executor pod SPARK-43504
        • +
      + +

      Structured Streaming

      + +
        +
      • Add support for tracking pinned blocks memory usage for RocksDB state store SPARK-43120
        • +
      • Add RocksDB state store provider memory management enhancements SPARK-43311
        • +
      • Introduce dropDuplicatesWithinWatermark SPARK-42931
        • +
      • Introduce a new callback onQueryIdle() to StreamingQueryListener SPARK-43183
        • +
      • Add option to skip commit coordinator as part of StreamingWrite API for DSv2 sources/sinks SPARK-42968
        • +
      • Introduce a new callback “onQueryIdle” to StreamingQueryListener SPARK-43183
        • +
      • Implement Changelog based Checkpointing for RocksDB State Store Provider SPARK-43421
        • +
      • Add support for WRITE_FLUSH_BYTES for RocksDB used in streaming stateful operators SPARK-42792
        • +
      • Add support for setting max_write_buffer_number and write_buffer_size for RocksDB used in streaming SPARK-42819
        • +
      • RocksDB StateStore lock acquisition should happen after getting input iterator from inputRDD SPARK-42566
        • +
      • Introduce watermark propagation among operators SPARK-42376
        • +
      • Cleanup orphan sst and log files in RocksDB checkpoint directory SPARK-42353
        • +
      • Expand QueryTerminatedEvent to contain error class if it exists in exception SPARK-43482
        • +
      + +

      ML

      + +
        +
      • Support Distributed Training of Functions Using Deepspeed SPARK-44264
        • +
      • Base interfaces of sparkML for spark3.5: estimator/transformer/model/evaluator SPARK-43516
        • +
      • Make MLv2 (ML on spark connect) supports pandas >= 2.0 SPARK-43783
        • +
      • Update MLv2 Transformer interfaces SPARK-43516
        • +
      • New pyspark ML logistic regression estimator implemented on top of distributor SPARK-43097
        • +
      • Add Classifier.getNumClasses back SPARK-42526
        • +
      • Write a Deepspeed Distributed Learning Class DeepspeedTorchDistributor SPARK-44264
        • +
      • Basic saving / loading implementation for ML on spark connect SPARK-43981
        • +
      • Improve logistic regression model saving SPARK-43097
        • +
      • Implement pipeline estimator for ML on spark connect SPARK-43982
        • +
      • Implement cross validator estimator SPARK-43983
        • +
      • Implement classification evaluator SPARK-44250
        • +
      • Make PyTorch Distributor compatible with Spark Connect SPARK-42993
        • +
      + +

      UI

      + +
        +
      • Add a Spark UI page for Spark Connect SPARK-44394
        • +
      • Support Heap Histogram column in Executors tab SPARK-44153
        • +
      • Show error message on UI for each failed query SPARK-44367
        • +
      • Display Add/Remove Time of Executors on Executors Tab SPARK-44309
        • +
      + +

      Build and Others

      + + + +

      Removals, Behavior Changes and Deprecations

      + +

      Upcoming Removal

      +

      The following features will be removed in the next Spark major release

      +
        +
      • Support for Java 8 and Java 11, and the minimal supported Java version will be Java 17
        • +
      • Support for Scala 2.12, and the minimal supported Scala version will be 2.13
        • +
      + +

      Migration Guides

      + + +

      Credits

      + +

      Last but not least, this release would not have been possible without the following contributors: +Adam Binford, Ahmed Hussein, Alex Jing, Alice Sayutina, Alkis Evlogimenos, Allan Folting, Allison Portis, Allison Wang, Amanda Liu, Andrey Gubichev, Andy Grove, Anish Shrigondekar, Anton Okolnychyi, Bartosz Konieczny, Beishao Cao, Bjørn Jørgensen, Bo Zhang, Bruce Robbins, Chandni Singh, Chao Sun, Chaoqin Li, Cheng Pan, Christopher Cooper, Christopher Watford, ConeyLiu, Daniel Sparing, Daniel Tenedorio, David Lewis, Dongjoon Hyun, Emil Ejbyfeldt, Enrico Minack, Eric Marnadi, Eric Ogren, Erik Krogen, Eugene Gusev, Fei Wang, Fokko Driesprong, Frank Yin, Fu Chen, Gene Pang, Gengliang Wang, Gera Shegalov, Giambattista Bloisi, Guilhem Vuillier, Gurpreet Singh, Haejoon Lee, Harsh Motwani, Henry Mai, Herman Van Hovell, Hisoka-X, Holden Karau, Huanli Wang, Hyukjin Kwon, Ismaël Mejía, Ivan Sadikov, Jack Chen, James Lamb, Jason Li, Jerry Peng, Jia Fan, Jiaan Geng, JinHelin404, Joe Wang, Johan Lasperas, John Zhuge, Josh Rosen, Juliusz Sompolski, Jungtaek Lim, Kazuyuki Tanimura, Kent Yao, Khalid Mammadov, Koray Beyaz, Kris Mok, Kun Wan, Kwafoor, Lee Yang, Leibnizhu, Liang Yan, Liang-Chi Hsieh, Lingkai Kong, Luca Canali, Ludovic Henry, Manu Zhang, Martin Grund, Mathew Jacob, Max Gekk, Menelaos Karavelas, Michael Chen, Michael Zhang, Mingkang Li, Mridul Muralidharan, NarekDW, Navin Viswanath, Niranjan Jayakar, Ole Sasse, Parth Upadhyay, Peter Toth, Qi Tan, Raghu Angadi, Richard Yu, Ritika Maheshwari, Rob Reeves, Robert Dillitz, Rui Wang, Ruifeng Zheng, Runyao Chen, Ryan Berti, Ryan Johnson, Serge Rielau, Serge Smertin, Sergii Druzkin, Shaoyun Chen, Shreyesh Shaju Arangath, Shrikant Prasad, Shu Wang, Siying Dong, Steve Vaughan Jr, Steve Weis, Steven Aerts, Steven Chen, Stove-hust, Szehon Ho, Takuya UESHIN, Tengfei Huang, Terry Kim, Thejdeep Gudivada, Thomas Graves, Tim Nieradzik, Venki Korukanti, Vihang Karajgaonkar, Vinod KC, Vitalii Li, Vsevolod Stepanov, Wanqiang Ji, Warren Zhu, Wei Liu, Weichen Xu, Wen Yuen Pang, Wenchen Fan, Willi Raschkowski, William Hyun, Xiduo You, Xieming LI, Xingbo Jiang, Xinrong Meng, Xinyi Yu, Yang Jie, Yi Zhu, Yihong He, Yikf, Yikun Jiang, Yiqun Zhang, Yohahaha, Yuanjian Li, Yuming Wang, Zhen Li, Ziqi Liu, advancedxy, alexanderwu-db, amousavigourabi, bjornjorgensen, bogao007, bowenliang123, caican00, chong0929, clownxc, frankliee, haoyanzhang, hdaly0, huangxiaopingRD, jdesjean, jerqi, jwang0306, khalidmammadov, kings129, kylerong-db, liangyu-1, lucaspompeun, mcdull-zhang, panbingkun, pegasas, pengzhon-db, ronandoolan2, smallzhongfeng, srielau, stijndehaes, sudoliyang, ted-jenks, tianhanhu, vicennial, wForget, wayne-kyungwonpark, wayneguow, yabola, zeruibao, zhouyifan279, zml1206, zzzzming95

      + + + +

      +
      +Spark News Archive +

      + +
      +
      +
      +
      Latest News
      + +

      Archive

      +
      +
      + + + +
      + +
      +
      + + + +
      +
      + Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered + trademarks or trademarks of The Apache Software Foundation in the United States and other countries. + See guidance on use of Apache Spark trademarks. + All other marks mentioned may be trademarks or registered trademarks of their respective owners. + Copyright © 2018 The Apache Software Foundation, Licensed under the + Apache License, Version 2.0. +
      +
      + + + + + + + diff --git a/site/research.html b/site/research.html index def6aad2981..7c5e578ec8b 100644 --- a/site/research.html +++ b/site/research.html @@ -187,6 +187,9 @@

      Spark Research

      Latest News

      Archive

      diff --git a/site/screencasts/1-first-steps-with-spark.html b/site/screencasts/1-first-steps-with-spark.html index 9ac317ed6a5..fc406b52ef4 100644 --- a/site/screencasts/1-first-steps-with-spark.html +++ b/site/screencasts/1-first-steps-with-spark.html @@ -149,6 +149,9 @@

      First Steps with Spark - Screencast #1

      Latest News

      Archive

      diff --git a/site/screencasts/2-spark-documentation-overview.html b/site/screencasts/2-spark-documentation-overview.html index 4903af54eda..8d83afc33b7 100644 --- a/site/screencasts/2-spark-documentation-overview.html +++ b/site/screencasts/2-spark-documentation-overview.html @@ -149,6 +149,9 @@

      Spark Documentation Overview – Screencast #2

      Latest News

      Archive

      diff --git a/site/screencasts/3-transformations-and-caching.html b/site/screencasts/3-transformations-and-caching.html index 9cabacdedf8..d31f5073ea1 100644 --- a/site/screencasts/3-transformations-and-caching.html +++ b/site/screencasts/3-transformations-and-caching.html @@ -145,6 +145,9 @@

      Transformations and Caching - Spark Screencast #3

      Latest News

      Archive

      diff --git a/site/screencasts/4-a-standalone-job-in-spark.html b/site/screencasts/4-a-standalone-job-in-spark.html index 1a71330e18a..356493e6598 100644 --- a/site/screencasts/4-a-standalone-job-in-spark.html +++ b/site/screencasts/4-a-standalone-job-in-spark.html @@ -143,6 +143,9 @@

      A Standalone Job in Scala - Spark Screencast #4

      Latest News

      Archive

      diff --git a/site/screencasts/index.html b/site/screencasts/index.html index 36e3c2d54f2..70779c261e5 100644 --- a/site/screencasts/index.html +++ b/site/screencasts/index.html @@ -171,6 +171,9 @@

      Fir
      Latest News

      Archive

      diff --git a/site/security.html b/site/security.html index 8a5894f8053..9c61825f3c2 100644 --- a/site/security.html +++ b/site/security.html @@ -650,6 +650,9 @@

      CVE-2017-7678 Apache Spark™

      Latest News

      Archive

      diff --git a/site/sitemap.xml b/site/sitemap.xml index 15d107fc98b..e064d52b2d4 100644 --- a/site/sitemap.xml +++ b/site/sitemap.xml @@ -143,6 +143,14 @@ 1.0 + + https://spark.apache.org/releases/spark-release-3-5-0.html + weekly + + + https://spark.apache.org/news/spark-3-5-0-released.html + weekly + https://spark.apache.org/releases/spark-release-3-3-3.html weekly @@ -1013,19 +1021,19 @@ weekly - https://spark.apache.org/news/ + https://spark.apache.org/streaming/ weekly - https://spark.apache.org/screencasts/ + https://spark.apache.org/news/ weekly - https://spark.apache.org/sql/ + https://spark.apache.org/screencasts/ weekly - https://spark.apache.org/streaming/ + https://spark.apache.org/sql/ weekly diff --git a/site/sql/index.html b/site/sql/index.html index 81de93c5e04..86d0e37ebd3 100644 --- a/site/sql/index.html +++ b/site/sql/index.html @@ -285,6 +285,9 @@

      Getting started

      Latest News

      Archive

      diff --git a/site/streaming/index.html b/site/streaming/index.html index 285a60d754d..42f50619dbd 100644 --- a/site/streaming/index.html +++ b/site/streaming/index.html @@ -227,6 +227,9 @@

      Community

      Latest News

      Archive

      diff --git a/site/third-party-projects.html b/site/third-party-projects.html index ce45cebd4f2..f018a3d8063 100644 --- a/site/third-party-projects.html +++ b/site/third-party-projects.html @@ -230,6 +230,9 @@

      Kotlin

      Latest News

      Archive

      diff --git a/site/trademarks.html b/site/trademarks.html index 4d589e64260..e2358d4bef9 100644 --- a/site/trademarks.html +++ b/site/trademarks.html @@ -189,6 +189,9 @@

      Apache Spark™Latest News

      Archive

      diff --git a/site/versioning-policy.html b/site/versioning-policy.html index a2d8035ff2b..e8f4c608361 100644 --- a/site/versioning-policy.html +++ b/site/versioning-policy.html @@ -291,6 +291,9 @@

      Maintenance releases and EOL

      Latest News

      Archive