From ca0128e1e829a9a8c9c2877a187aed1f79831d35 Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Tue, 12 Jun 2018 11:09:01 -0700 Subject: [PATCH 1/3] Update list of mirrored releases, add more instructions to release guide. --- js/downloads.js | 54 +++++++++++++++++++-------------------- release-process.md | 9 ++++++- site/doap.rdf | 5 +++- site/js/downloads.js | 54 +++++++++++++++++++-------------------- site/release-process.html | 8 +++++- 5 files changed, 71 insertions(+), 59 deletions(-) diff --git a/js/downloads.js b/js/downloads.js index 80dbdcbb834..d817dbd6e31 100644 --- a/js/downloads.js +++ b/js/downloads.js @@ -3,8 +3,13 @@ releases = {}; -function addRelease(version, releaseDate, packages, stable) { - releases[version] = {released: releaseDate, packages: packages, stable: stable}; +function addRelease(version, releaseDate, packages, stable, mirrored) { + releases[version] = { + released: releaseDate, + packages: packages, + stable: stable, + mirrored: mirrored + }; } var sources = {pretty: "Source Code", tag: "sources"}; @@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source // 2.2.0+ var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources]; -addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true); -addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true); -addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true); -addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true); -addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true); -addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true); -addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true); -addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true); -addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true); -addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true); +addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true); +addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true); +addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true); +addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false); +addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true); +addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false); +addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false); +addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true); +addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false); +addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false); //addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false); -addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true); -addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true); -addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true); -addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true); -//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true); -//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true); -//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true); -//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true); -//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true); +addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true); +addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false); +addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false); +addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false); function append(el, contents) { el.innerHTML += contents; @@ -128,10 +128,10 @@ function onVersionSelect() { append(verifyLink, link); // Populate releases - updateDownloadLink(); + updateDownloadLink(releases[version].mirrored); } -function updateDownloadLink() { +function updateDownloadLink(isMirrored) { var versionSelect = document.getElementById("sparkVersionSelect"); var packageSelect = document.getElementById("sparkPackageSelect"); var downloadLink = document.getElementById("spanDownloadLink"); @@ -147,12 +147,10 @@ function updateDownloadLink() { .replace(/-bin-sources/, ""); // special case for source packages var link = ""; - if (version < "1.6.3" || - (version >= "2.0.0" && version <= "2.0.1") || - (version >= "2.1.0" && version <= "2.1.1")) { - link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact"; - } else { + if (isMirrored) { link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact"; + } else { + link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact"; } link = link .replace(/\$ver/, version) diff --git a/release-process.md b/release-process.md index e756cc0b56a..8cd38521de6 100644 --- a/release-process.md +++ b/release-process.md @@ -194,9 +194,16 @@ You should also delete the RC directories from the staging repository. For examp svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \ https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \ -m"Removing RC artifacts." - ``` +When removing older releases, also update `js/download.js` to indicate the release is not mirrored +anymore, so that the correct links are generated on the site. + +Also take a moment to check `HiveExternalCatalogVersionsSuite.scala` starting with branch-2.2 +and see if it needs to be adjusted, since that test relies on mirrored downloads of previous +releases. + +

Update the Spark Apache Repository

Check out the tagged commit for the release candidate that passed and apply the correct version tag. diff --git a/site/doap.rdf b/site/doap.rdf index 1568c3fdea8..25bb8fc4be5 100644 --- a/site/doap.rdf +++ b/site/doap.rdf @@ -28,13 +28,16 @@ Apache Spark is a fast and general engine for large-scale data processing. - Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala and Python as well as a rich set of libraries including stream processing, machine learning, and graph analytics. + Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala, Python and R, as well as a rich set of libraries including stream processing, machine learning, and graph analytics. Java Scala Python + R + SQL + JDBC diff --git a/site/js/downloads.js b/site/js/downloads.js index 80dbdcbb834..d817dbd6e31 100644 --- a/site/js/downloads.js +++ b/site/js/downloads.js @@ -3,8 +3,13 @@ releases = {}; -function addRelease(version, releaseDate, packages, stable) { - releases[version] = {released: releaseDate, packages: packages, stable: stable}; +function addRelease(version, releaseDate, packages, stable, mirrored) { + releases[version] = { + released: releaseDate, + packages: packages, + stable: stable, + mirrored: mirrored + }; } var sources = {pretty: "Source Code", tag: "sources"}; @@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source // 2.2.0+ var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources]; -addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true); -addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true); -addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true); -addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true); -addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true); -addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true); -addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true); -addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true); -addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true); -addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true); +addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true); +addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true); +addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true); +addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false); +addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true); +addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false); +addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false); +addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true); +addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false); +addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false); //addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false); -addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true); -addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true); -addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true); -addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true); -//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true); -//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true); -//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true); -//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true); -//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true); +addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true); +addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false); +addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false); +addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false); function append(el, contents) { el.innerHTML += contents; @@ -128,10 +128,10 @@ function onVersionSelect() { append(verifyLink, link); // Populate releases - updateDownloadLink(); + updateDownloadLink(releases[version].mirrored); } -function updateDownloadLink() { +function updateDownloadLink(isMirrored) { var versionSelect = document.getElementById("sparkVersionSelect"); var packageSelect = document.getElementById("sparkPackageSelect"); var downloadLink = document.getElementById("spanDownloadLink"); @@ -147,12 +147,10 @@ function updateDownloadLink() { .replace(/-bin-sources/, ""); // special case for source packages var link = ""; - if (version < "1.6.3" || - (version >= "2.0.0" && version <= "2.0.1") || - (version >= "2.1.0" && version <= "2.1.1")) { - link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact"; - } else { + if (isMirrored) { link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact"; + } else { + link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact"; } link = link .replace(/\$ver/, version) diff --git a/site/release-process.html b/site/release-process.html index a3b311d6981..8151488fb2f 100644 --- a/site/release-process.html +++ b/site/release-process.html @@ -391,9 +391,15 @@

Remove Old Releases from Development Repository and Mirror Network

svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
   https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
   -m"Removing RC artifacts."
-
+

When removing older releases, also update js/download.js to indicate the release is not mirrored +anymore, so that the correct links are generated on the site.

+ +

Also take a moment to check HiveExternalCatalogVersionsSuite.scala starting with branch-2.2 +and see if it needs to be adjusted, since that test relies on mirrored downloads of previous +releases.

+

Update the Spark Apache Repository

Check out the tagged commit for the release candidate that passed and apply the correct version tag.

From e963c61f04bafe9948f87b895a731dbac484990b Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Tue, 12 Jun 2018 14:50:26 -0700 Subject: [PATCH 2/3] More stuff. --- release-process.md | 2 ++ site/release-process.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/release-process.md b/release-process.md index 8cd38521de6..0b2d936fe84 100644 --- a/release-process.md +++ b/release-process.md @@ -196,6 +196,8 @@ svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \ -m"Removing RC artifacts." ``` +Make sure to also remove the unpublished staging repositories from the Apache Nexus Repository Manager. + When removing older releases, also update `js/download.js` to indicate the release is not mirrored anymore, so that the correct links are generated on the site. diff --git a/site/release-process.html b/site/release-process.html index 8151488fb2f..dd62edcee28 100644 --- a/site/release-process.html +++ b/site/release-process.html @@ -393,6 +393,8 @@

Remove Old Releases from Development Repository and Mirror Network

-m"Removing RC artifacts." +

Make sure to also remove the unpublished staging repositories from the Apache Nexus Repository Manager.

+

When removing older releases, also update js/download.js to indicate the release is not mirrored anymore, so that the correct links are generated on the site.

From 58e92371ee30e3a3aea212f1a51144945e9b7a2b Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Wed, 13 Jun 2018 13:40:48 -0700 Subject: [PATCH 3/3] Better guidance for what to delete. --- release-process.md | 29 ++++++++++++++++------------- site/release-process.html | 26 ++++++++++++++------------ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/release-process.md b/release-process.md index 0b2d936fe84..b2ee62ba98d 100644 --- a/release-process.md +++ b/release-process.md @@ -177,18 +177,10 @@ Publishing to CRAN is done using PMC. -

Remove Old Releases from Development Repository and Mirror Network

+

Remove RC artifacts from repositories

-Spark always keeps two releases in the mirror network: the most recent release on the current and -previous branches. To delete older versions simply use svn rm. The `downloads.js` file in the -website `js/` directory must also be updated to reflect the changes. For instance, the two -releases should be 1.1.1 and 1.0.2, but not 1.1.1 and 1.1.0. - -``` -$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0 -``` - -You should also delete the RC directories from the staging repository. For example: +After the vote passes and you moved the approved RC to the release repository, you should delete +the RC directories from the staging repository. For example: ``` svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \ @@ -196,9 +188,20 @@ svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \ -m"Removing RC artifacts." ``` -Make sure to also remove the unpublished staging repositories from the Apache Nexus Repository Manager. +Make sure to also remove the unpublished staging repositories from the +Apache Nexus Repository Manager. + + +

Remove Old Releases from Mirror Network

+ +Spark always keeps the latest maintance released of each branch in the mirror network. +To delete older versions simply use svn rm: + +``` +$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0 +``` -When removing older releases, also update `js/download.js` to indicate the release is not mirrored +You will also need to update `js/download.js` to indicate the release is not mirrored anymore, so that the correct links are generated on the site. Also take a moment to check `HiveExternalCatalogVersionsSuite.scala` starting with branch-2.2 diff --git a/site/release-process.html b/site/release-process.html index dd62edcee28..24157989106 100644 --- a/site/release-process.html +++ b/site/release-process.html @@ -376,26 +376,28 @@

Publish to CRAN

Publishing to CRAN is done using this form. Since it requires further manual steps, please also contact the PMC.

-

Remove Old Releases from Development Repository and Mirror Network

+

Remove RC artifacts from repositories

-

Spark always keeps two releases in the mirror network: the most recent release on the current and -previous branches. To delete older versions simply use svn rm. The downloads.js file in the -website js/ directory must also be updated to reflect the changes. For instance, the two -releases should be 1.1.1 and 1.0.2, but not 1.1.1 and 1.1.0.

- -
$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
-
- -

You should also delete the RC directories from the staging repository. For example:

+

After the vote passes and you moved the approved RC to the release repository, you should delete +the RC directories from the staging repository. For example:

svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
   https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
   -m"Removing RC artifacts."
-

Make sure to also remove the unpublished staging repositories from the Apache Nexus Repository Manager.

+

Make sure to also remove the unpublished staging repositories from the +Apache Nexus Repository Manager.

+ +

Remove Old Releases from Mirror Network

+ +

Spark always keeps the latest maintance released of each branch in the mirror network. +To delete older versions simply use svn rm:

+ +
$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
+
-

When removing older releases, also update js/download.js to indicate the release is not mirrored +

You will also need to update js/download.js to indicate the release is not mirrored anymore, so that the correct links are generated on the site.

Also take a moment to check HiveExternalCatalogVersionsSuite.scala starting with branch-2.2