From d118082f6146996a646446808f2566a719d189b9 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 14:52:35 +0900 Subject: [PATCH 1/4] Spark build automation --- appveyor.yml | 47 ++++++++++ dev/appveyor-install-dependencies.ps1 | 126 ++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 appveyor.yml create mode 100644 dev/appveyor-install-dependencies.ps1 diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000000..92d890c625fe --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "{build}-{branch}" + +shallow_clone: true + +platform: x64 +configuration: Debug + +only_commits: + files: + - R/ + +cache: + - C:\Users\appveyor\.m2 + +install: + # Install maven and dependencies + - ps: .\dev\appveyor-install-dependencies.ps1 + # Required package for R unit tests + - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" + +build_script: + - cmd: mvn -DskipTests -Psparkr -Phive -Phive-thriftserver package + +test_script: + - cmd: .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///" R\pkg\tests\run-all.R + +notifications: + - provider: Email + on_build_success: false + on_build_failure: false + on_build_status_changed: false + diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 new file mode 100644 index 000000000000..087b8666cc68 --- /dev/null +++ b/dev/appveyor-install-dependencies.ps1 @@ -0,0 +1,126 @@ +<# +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +#> + +$CRAN = "https://cloud.r-project.org" + +Function InstallR { + if ( -not(Test-Path Env:\R_ARCH) ) { + $arch = "i386" + } + Else { + $arch = $env:R_ARCH + } + + $urlPath = "" + $latestVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + If ($rVer -ne $latestVer) { + $urlPath = ("old/" + $rVer + "/") + } + + $rurl = $CRAN + "/bin/windows/base/" + $urlPath + "R-" + $rVer + "-win.exe" + + # Downloading R + Start-FileDownload $rurl "R-win.exe" + + # Running R installer + Start-Process -FilePath .\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait + + $RDrive = "C:" + echo "R is now available on drive $RDrive" + + $env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH + + # Testing R installation + Rscript -e "sessionInfo()" +} + +Function InstallRtools { + $rtoolsver = $rToolsVer.Split('.')[0..1] -Join '' + $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" + + # Downloading Rtools + Start-FileDownload $rtoolsurl "Rtools-current.exe" + + # Running Rtools installer + Start-Process -FilePath .\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait + + $RtoolsDrive = "C:" + echo "Rtools is now available on drive $RtoolsDrive" + + if ( -not(Test-Path Env:\GCC_PATH) ) { + $gccPath = "gcc-4.6.3" + } + Else { + $gccPath = $env:GCC_PATH + } + $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gccPath + '\bin;' + $env:PATH + $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/' +} + +# create tools directory outside of Spark directory +$up = (Get-Item -Path ".." -Verbose).FullName +$tools = "$up\tools" +if (!(Test-Path $tools)) { + New-Item -ItemType Directory -Force -Path $tools | Out-Null +} + +# ========================== Maven +Push-Location $tools + +$mavenVer = "3.3.9" +Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip" + +# extract +Invoke-Expression "7z.exe x maven.zip" + +# add maven to environment variables +$env:Path += ";$tools\apache-maven-$mavenVer\bin" +$env:M2_HOME = "$tools\apache-maven-$mavenVer" +$env:MAVEN_OPTS = "-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m" + +Pop-Location + +# ========================== Hadoop bin package +$hadoopVer = "2.6.0" +$hadoopPath = "$tools\hadoop" +if (!(Test-Path $hadoopPath)) { + New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null +} +Push-Location $hadoopPath + +Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip" + +# extract +Invoke-Expression "7z.exe x winutils-master.zip" + +# add hadoop bin to environment variables +$env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" + +Pop-Location + +# ========================== R +$rVer = "3.3.1" +$rToolsVer = "3.4.0" + +InstallR +InstallRtools + +$env:R_LIBS_USER = 'c:\RLibrary' +if ( -not(Test-Path $env:R_LIBS_USER) ) { + mkdir $env:R_LIBS_USER +} + From b29f7ba3fda095ddb424edd568111ed86d810cbc Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 14:57:56 +0900 Subject: [PATCH 2/4] Change in R test --- R/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/README.md b/R/README.md index 932d5272d0b4..0d82ff356fdb 100644 --- a/R/README.md +++ b/R/README.md @@ -1,4 +1,4 @@ -# R on Spark +# R on Apache Spark SparkR is an R package that provides a light-weight frontend to use Spark from R. From b5681d68f78b8e0ee2ef6531633f5b87e35d0ab5 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 15:30:48 +0900 Subject: [PATCH 3/4] Run builds only with master --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 92d890c625fe..8950826101fc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,6 +20,10 @@ shallow_clone: true platform: x64 configuration: Debug +branches: + only: + - master + only_commits: files: - R/ From 1e458cc2ae727ba9a74a17f40c1033e28dce4d0a Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 15:32:46 +0900 Subject: [PATCH 4/4] Changes in R --- R/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/README.md b/R/README.md index 0d82ff356fdb..932d5272d0b4 100644 --- a/R/README.md +++ b/R/README.md @@ -1,4 +1,4 @@ -# R on Apache Spark +# R on Spark SparkR is an R package that provides a light-weight frontend to use Spark from R.