From cf9f56c8571f947b5e89e741a2b5f2f5477843b2 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 16:24:18 +0900 Subject: [PATCH 01/31] Appveyor SparkR Windows test draft --- appveyor.yml | 45 +++++++ dev/appveyor-install-dependencies.ps1 | 60 +++++++++ dev/appveyor-install-r.ps1 | 167 ++++++++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 appveyor.yml create mode 100644 dev/appveyor-install-dependencies.ps1 create mode 100644 dev/appveyor-install-r.ps1 diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000000..8eb78c100083 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "{build}-{branch}" + +shallow_clone: true + +platform: x64 +configuration: Debug + +cache: + - C:\Users\appveyor\.m2 + +# Install maven and dependencies +install: + - ps: | + Import-Module '.\scripts\appveyor-tool.ps1' + Bootstrap + - ps: .\dev\appveyor-install-dependencies.ps1 + - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" + +build_script: + - cmd: mvn -DskipTests -Psparkr package + +test_script: + - cmd: .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///" R\pkg\tests\run-all.R + +notifications: + - provider: Email + on_build_success: false + on_build_failure: false + on_build_status_changed: false + diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 new file mode 100644 index 000000000000..3ebaa1e131f9 --- /dev/null +++ b/dev/appveyor-install-dependencies.ps1 @@ -0,0 +1,60 @@ +<# +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +#> + +# create tools directory outside of REEF directory +$up = (Get-Item -Path ".." -Verbose).FullName +$tools = "$up\tools" +if (!(Test-Path $tools)) +{ + New-Item -ItemType Directory -Force -Path $tools | Out-Null +} + +# ========================== maven +Push-Location $tools + +$mavenVer = "3.3.3" +Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip" + +# extract +Invoke-Expression "7z.exe x maven.zip" | Out-Null + +# add maven to environment variables +$env:Path += ";$tools\apache-maven-$mavenVer\bin" +$env:M2_HOME = "$tools\apache-maven-$mavenVer" +$env:MAVEN_OPTS = "-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m" + +Pop-Location + +# ========================== Hadoop bin package +$hadoopVer = "2.6.0" +$hadoopPath = "$tools\hadoop" +if (!(Test-Path $hadoopPath)) +{ + New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null +} +Push-Location $hadoopPath + +Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip" + +# extract +Invoke-Expression "7z.exe x winutils-master.zip" | Out-Null + +# add hadoop bin to environment variables +$env:HADOOP_HOME = "$hadoopPath/hadoop-$hadoopVer" + +Pop-Location + diff --git a/dev/appveyor-install-r.ps1 b/dev/appveyor-install-r.ps1 new file mode 100644 index 000000000000..14e00531c8b3 --- /dev/null +++ b/dev/appveyor-install-r.ps1 @@ -0,0 +1,167 @@ +$CRAN = "https://cloud.r-project.org" + +# Found at http://zduck.com/2012/powershell-batch-files-exit-codes/ +Function Exec +{ + [CmdletBinding()] + param ( + [Parameter(Position=0, Mandatory=1)] + [scriptblock]$Command, + [Parameter(Position=1, Mandatory=0)] + [string]$ErrorMessage = "Execution of command failed.`n$Command" + ) + $ErrorActionPreference = "Continue" + & $Command 2>&1 | %{ "$_" } + if ($LastExitCode -ne 0) { + throw "Exec: $ErrorMessage`nExit code: $LastExitCode" + } +} + +Function Progress +{ + [CmdletBinding()] + param ( + [Parameter(Position=0, Mandatory=0)] + [string]$Message = "" + ) + + $ProgressMessage = '== ' + (Get-Date) + ': ' + $Message + + Write-Host $ProgressMessage -ForegroundColor Magenta +} + +Function InstallR { + [CmdletBinding()] + Param() + + if ( -not(Test-Path Env:\R_VERSION) ) { + $version = "patched" + } + Else { + $version = $env:R_VERSION + } + + if ( -not(Test-Path Env:\R_ARCH) ) { + $arch = "i386" + } + Else { + $arch = $env:R_ARCH + } + + Progress ("Version: " + $version) + + If ($version -eq "devel") { + $url_path = "" + $version = "devel" + } + ElseIf (($version -eq "stable") -or ($version -eq "release")) { + $url_path = "" + $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + If ($version -eq "3.2.4") { + $version = "3.2.4revised" + } + } + ElseIf ($version -eq "patched") { + $url_path = "" + $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + "patched" + } + ElseIf ($version -eq "oldrel") { + $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-oldrel).Content).version + $url_path = ("old/" + $version + "/") + } + Else { + $url_path = ("old/" + $version + "/") + } + + Progress ("URL path: " + $url_path) + + $rurl = $CRAN + "/bin/windows/base/" + $url_path + "R-" + $version + "-win.exe" + + Progress ("Downloading R from: " + $rurl) + Exec { bash -c ("curl --silent -o ../R-win.exe -L " + $rurl) } + + Progress "Running R installer" + Start-Process -FilePath ..\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait + + $RDrive = "C:" + echo "R is now available on drive $RDrive" + + Progress "Setting PATH" + $env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH + + Progress "Testing R installation" + Rscript -e "sessionInfo()" +} + +Function InstallRtools { + if ( -not(Test-Path Env:\RTOOLS_VERSION) ) { + Progress "Determining Rtools version" + $rtoolsver = $(Invoke-WebRequest ($CRAN + "/bin/windows/Rtools/VERSION.txt")).Content.Split(' ')[2].Split('.')[0..1] -Join '' + } + Else { + $rtoolsver = $env:RTOOLS_VERSION + } + + $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" + + Progress ("Downloading Rtools from: " + $rtoolsurl) + bash -c ("curl --silent -o ../Rtools-current.exe -L " + $rtoolsurl) + + Progress "Running Rtools installer" + Start-Process -FilePath ..\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait + + $RtoolsDrive = "C:" + echo "Rtools is now available on drive $RtoolsDrive" + + Progress "Setting PATH" + if ( -not(Test-Path Env:\GCC_PATH) ) { + $gcc_path = "gcc-4.6.3" + } + Else { + $gcc_path = $env:GCC_PATH + } + $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gcc_path + '\bin;' + $env:PATH + $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/' +} + +Function Bootstrap { + [CmdletBinding()] + Param() + + Progress "Bootstrap: Start" + + Progress "Adding GnuWin32 tools to PATH" + $env:PATH = "C:\Program Files (x86)\Git\bin;" + $env:PATH + + Progress "Setting time zone" + tzutil /g + tzutil /s "GMT Standard Time" + tzutil /g + + InstallR + + if ((Test-Path "src") -or ($env:USE_RTOOLS)) { + InstallRtools + } + Else { + Progress "Skipping download of Rtools because src/ directory is missing." + } + + Progress "Downloading and installing travis-tool.sh" + Invoke-WebRequest https://raw.github.com/krlmlr/r-appveyor/master/r-travis/scripts/travis-tool.sh -OutFile "..\travis-tool.sh" + echo '@bash.exe ../travis-tool.sh %*' | Out-File -Encoding ASCII .\travis-tool.sh.cmd + cat .\travis-tool.sh.cmd + bash -c "echo '^travis-tool\.sh\.cmd$' >> .Rbuildignore" + cat .\.Rbuildignore + + $env:PATH.Split(";") + + Progress "Setting R_LIBS_USER" + $env:R_LIBS_USER = 'c:\RLibrary' + if ( -not(Test-Path $env:R_LIBS_USER) ) { + mkdir $env:R_LIBS_USER + } + + Progress "Bootstrap: Done" +} + From b25eabaa88c4ada63e2d5a1bb802a48662f60d04 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 16:26:18 +0900 Subject: [PATCH 02/31] Fix script path for R installation --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 8eb78c100083..f6a9e4931718 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ cache: # Install maven and dependencies install: - ps: | - Import-Module '.\scripts\appveyor-tool.ps1' + Import-Module '.\dev\appveyor-tool.ps1' Bootstrap - ps: .\dev\appveyor-install-dependencies.ps1 - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" From 2772002e832570ccb7ac2120a2775a2daf724832 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 16:27:49 +0900 Subject: [PATCH 03/31] Fix the name of script for R installation --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index f6a9e4931718..a92c0615a1e7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ cache: # Install maven and dependencies install: - ps: | - Import-Module '.\dev\appveyor-tool.ps1' + Import-Module '.\dev\appveyor-install-r.ps1' Bootstrap - ps: .\dev\appveyor-install-dependencies.ps1 - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" From 6cb4416a5afe66c624602b6ff36c5d23dc32b62f Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 16:31:10 +0900 Subject: [PATCH 04/31] Upgrade maven version to 3.3.9 --- dev/appveyor-install-dependencies.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 3ebaa1e131f9..91168b8585e7 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -26,7 +26,7 @@ if (!(Test-Path $tools)) # ========================== maven Push-Location $tools -$mavenVer = "3.3.3" +$mavenVer = "3.3.9" Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip" # extract From a2852a0f1b40d715eaf00947acdb076727e2d42e Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 17:08:11 +0900 Subject: [PATCH 05/31] Clean up and fix the path for Hadoop bin package --- appveyor.yml | 4 +- dev/appveyor-install-dependencies.ps1 | 6 +-- dev/appveyor-install-r.ps1 | 54 +++++++++++---------------- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a92c0615a1e7..cf67cb3f473a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -25,9 +25,7 @@ cache: # Install maven and dependencies install: - - ps: | - Import-Module '.\dev\appveyor-install-r.ps1' - Bootstrap + - ps: .\dev\appveyor-install-r.ps1 - ps: .\dev\appveyor-install-dependencies.ps1 - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 91168b8585e7..d3da0e73b5d0 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -30,7 +30,7 @@ $mavenVer = "3.3.9" Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip" # extract -Invoke-Expression "7z.exe x maven.zip" | Out-Null +Invoke-Expression "7z.exe x maven.zip" # add maven to environment variables $env:Path += ";$tools\apache-maven-$mavenVer\bin" @@ -51,10 +51,10 @@ Push-Location $hadoopPath Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip" # extract -Invoke-Expression "7z.exe x winutils-master.zip" | Out-Null +Invoke-Expression "7z.exe x winutils-master.zip" # add hadoop bin to environment variables -$env:HADOOP_HOME = "$hadoopPath/hadoop-$hadoopVer" +$env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" Pop-Location diff --git a/dev/appveyor-install-r.ps1 b/dev/appveyor-install-r.ps1 index 14e00531c8b3..d490022824d7 100644 --- a/dev/appveyor-install-r.ps1 +++ b/dev/appveyor-install-r.ps1 @@ -124,44 +124,34 @@ Function InstallRtools { $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/' } -Function Bootstrap { - [CmdletBinding()] - Param() - - Progress "Bootstrap: Start" +[CmdletBinding()] +Param() - Progress "Adding GnuWin32 tools to PATH" - $env:PATH = "C:\Program Files (x86)\Git\bin;" + $env:PATH +Progress "Bootstrap: Start" - Progress "Setting time zone" - tzutil /g - tzutil /s "GMT Standard Time" - tzutil /g +Progress "Adding GnuWin32 tools to PATH" +$env:PATH = "C:\Program Files (x86)\Git\bin;" + $env:PATH - InstallR - - if ((Test-Path "src") -or ($env:USE_RTOOLS)) { - InstallRtools - } - Else { - Progress "Skipping download of Rtools because src/ directory is missing." - } +Progress "Setting time zone" +tzutil /g +tzutil /s "GMT Standard Time" +tzutil /g - Progress "Downloading and installing travis-tool.sh" - Invoke-WebRequest https://raw.github.com/krlmlr/r-appveyor/master/r-travis/scripts/travis-tool.sh -OutFile "..\travis-tool.sh" - echo '@bash.exe ../travis-tool.sh %*' | Out-File -Encoding ASCII .\travis-tool.sh.cmd - cat .\travis-tool.sh.cmd - bash -c "echo '^travis-tool\.sh\.cmd$' >> .Rbuildignore" - cat .\.Rbuildignore +InstallR - $env:PATH.Split(";") +if ((Test-Path "src") -or ($env:USE_RTOOLS)) { + InstallRtools +} +Else { + Progress "Skipping download of Rtools because src/ directory is missing." +} - Progress "Setting R_LIBS_USER" - $env:R_LIBS_USER = 'c:\RLibrary' - if ( -not(Test-Path $env:R_LIBS_USER) ) { - mkdir $env:R_LIBS_USER - } +$env:PATH.Split(";") - Progress "Bootstrap: Done" +Progress "Setting R_LIBS_USER" +$env:R_LIBS_USER = 'c:\RLibrary' +if ( -not(Test-Path $env:R_LIBS_USER) ) { + mkdir $env:R_LIBS_USER } +Progress "Bootstrap: Done" From 5aca1045ccfd7eed35bd8fa7721867c363bd88a0 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 17:18:55 +0900 Subject: [PATCH 06/31] Merged dependecies installation --- appveyor.yml | 1 - dev/appveyor-install-dependencies.ps1 | 145 +++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cf67cb3f473a..a5ba9510f418 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -25,7 +25,6 @@ cache: # Install maven and dependencies install: - - ps: .\dev\appveyor-install-r.ps1 - ps: .\dev\appveyor-install-dependencies.ps1 - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index d3da0e73b5d0..339b200a8293 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -15,7 +15,133 @@ specific language governing permissions and limitations under the License. #> -# create tools directory outside of REEF directory +$CRAN = "https://cloud.r-project.org" + +# Found at http://zduck.com/2012/powershell-batch-files-exit-codes/ +Function Exec +{ + [CmdletBinding()] + param ( + [Parameter(Position=0, Mandatory=1)] + [scriptblock]$Command, + [Parameter(Position=1, Mandatory=0)] + [string]$ErrorMessage = "Execution of command failed.`n$Command" + ) + $ErrorActionPreference = "Continue" + & $Command 2>&1 | %{ "$_" } + if ($LastExitCode -ne 0) { + throw "Exec: $ErrorMessage`nExit code: $LastExitCode" + } +} + +Function Progress +{ + [CmdletBinding()] + param ( + [Parameter(Position=0, Mandatory=0)] + [string]$Message = "" + ) + + $ProgressMessage = '== ' + (Get-Date) + ': ' + $Message + + Write-Host $ProgressMessage -ForegroundColor Magenta +} + +Function InstallR { + [CmdletBinding()] + Param() + + if ( -not(Test-Path Env:\R_VERSION) ) { + $version = "patched" + } + Else { + $version = $env:R_VERSION + } + + if ( -not(Test-Path Env:\R_ARCH) ) { + $arch = "i386" + } + Else { + $arch = $env:R_ARCH + } + + Progress ("Version: " + $version) + + If ($version -eq "devel") { + $url_path = "" + $version = "devel" + } + ElseIf (($version -eq "stable") -or ($version -eq "release")) { + $url_path = "" + $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + If ($version -eq "3.2.4") { + $version = "3.2.4revised" + } + } + ElseIf ($version -eq "patched") { + $url_path = "" + $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + "patched" + } + ElseIf ($version -eq "oldrel") { + $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-oldrel).Content).version + $url_path = ("old/" + $version + "/") + } + Else { + $url_path = ("old/" + $version + "/") + } + + Progress ("URL path: " + $url_path) + + $rurl = $CRAN + "/bin/windows/base/" + $url_path + "R-" + $version + "-win.exe" + + Progress ("Downloading R from: " + $rurl) + Exec { bash -c ("curl --silent -o ../R-win.exe -L " + $rurl) } + + Progress "Running R installer" + Start-Process -FilePath ..\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait + + $RDrive = "C:" + echo "R is now available on drive $RDrive" + + Progress "Setting PATH" + $env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH + + Progress "Testing R installation" + Rscript -e "sessionInfo()" +} + +Function InstallRtools { + if ( -not(Test-Path Env:\RTOOLS_VERSION) ) { + Progress "Determining Rtools version" + $rtoolsver = $(Invoke-WebRequest ($CRAN + "/bin/windows/Rtools/VERSION.txt")).Content.Split(' ')[2].Split('.')[0..1] -Join '' + } + Else { + $rtoolsver = $env:RTOOLS_VERSION + } + + $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" + + Progress ("Downloading Rtools from: " + $rtoolsurl) + bash -c ("curl --silent -o ../Rtools-current.exe -L " + $rtoolsurl) + + Progress "Running Rtools installer" + Start-Process -FilePath ..\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait + + $RtoolsDrive = "C:" + echo "Rtools is now available on drive $RtoolsDrive" + + Progress "Setting PATH" + if ( -not(Test-Path Env:\GCC_PATH) ) { + $gcc_path = "gcc-4.6.3" + } + Else { + $gcc_path = $env:GCC_PATH + } + $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gcc_path + '\bin;' + $env:PATH + $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/' +} + +# create tools directory outside of Spark directory $up = (Get-Item -Path ".." -Verbose).FullName $tools = "$up\tools" if (!(Test-Path $tools)) @@ -39,7 +165,7 @@ $env:MAVEN_OPTS = "-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m" Pop-Location -# ========================== Hadoop bin package +# ========================== Hadoop bin package $hadoopVer = "2.6.0" $hadoopPath = "$tools\hadoop" if (!(Test-Path $hadoopPath)) @@ -58,3 +184,18 @@ $env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" Pop-Location +# ========================== R +$env:PATH = "C:\Program Files (x86)\Git\bin;" + $env:PATH +InstallR +if ((Test-Path "src") -or ($env:USE_RTOOLS)) { + InstallRtools +} +Else { + Progress "Skipping download of Rtools because src/ directory is missing." +} +$env:PATH.Split(";") +$env:R_LIBS_USER = 'c:\RLibrary' +if ( -not(Test-Path $env:R_LIBS_USER) ) { + mkdir $env:R_LIBS_USER +} + From fbcfe135db8e6515f3dadccce7c2d72f6dec9b91 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 17:19:46 +0900 Subject: [PATCH 07/31] Remove R installation script --- dev/appveyor-install-r.ps1 | 157 ------------------------------------- 1 file changed, 157 deletions(-) delete mode 100644 dev/appveyor-install-r.ps1 diff --git a/dev/appveyor-install-r.ps1 b/dev/appveyor-install-r.ps1 deleted file mode 100644 index d490022824d7..000000000000 --- a/dev/appveyor-install-r.ps1 +++ /dev/null @@ -1,157 +0,0 @@ -$CRAN = "https://cloud.r-project.org" - -# Found at http://zduck.com/2012/powershell-batch-files-exit-codes/ -Function Exec -{ - [CmdletBinding()] - param ( - [Parameter(Position=0, Mandatory=1)] - [scriptblock]$Command, - [Parameter(Position=1, Mandatory=0)] - [string]$ErrorMessage = "Execution of command failed.`n$Command" - ) - $ErrorActionPreference = "Continue" - & $Command 2>&1 | %{ "$_" } - if ($LastExitCode -ne 0) { - throw "Exec: $ErrorMessage`nExit code: $LastExitCode" - } -} - -Function Progress -{ - [CmdletBinding()] - param ( - [Parameter(Position=0, Mandatory=0)] - [string]$Message = "" - ) - - $ProgressMessage = '== ' + (Get-Date) + ': ' + $Message - - Write-Host $ProgressMessage -ForegroundColor Magenta -} - -Function InstallR { - [CmdletBinding()] - Param() - - if ( -not(Test-Path Env:\R_VERSION) ) { - $version = "patched" - } - Else { - $version = $env:R_VERSION - } - - if ( -not(Test-Path Env:\R_ARCH) ) { - $arch = "i386" - } - Else { - $arch = $env:R_ARCH - } - - Progress ("Version: " + $version) - - If ($version -eq "devel") { - $url_path = "" - $version = "devel" - } - ElseIf (($version -eq "stable") -or ($version -eq "release")) { - $url_path = "" - $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version - If ($version -eq "3.2.4") { - $version = "3.2.4revised" - } - } - ElseIf ($version -eq "patched") { - $url_path = "" - $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + "patched" - } - ElseIf ($version -eq "oldrel") { - $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-oldrel).Content).version - $url_path = ("old/" + $version + "/") - } - Else { - $url_path = ("old/" + $version + "/") - } - - Progress ("URL path: " + $url_path) - - $rurl = $CRAN + "/bin/windows/base/" + $url_path + "R-" + $version + "-win.exe" - - Progress ("Downloading R from: " + $rurl) - Exec { bash -c ("curl --silent -o ../R-win.exe -L " + $rurl) } - - Progress "Running R installer" - Start-Process -FilePath ..\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait - - $RDrive = "C:" - echo "R is now available on drive $RDrive" - - Progress "Setting PATH" - $env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH - - Progress "Testing R installation" - Rscript -e "sessionInfo()" -} - -Function InstallRtools { - if ( -not(Test-Path Env:\RTOOLS_VERSION) ) { - Progress "Determining Rtools version" - $rtoolsver = $(Invoke-WebRequest ($CRAN + "/bin/windows/Rtools/VERSION.txt")).Content.Split(' ')[2].Split('.')[0..1] -Join '' - } - Else { - $rtoolsver = $env:RTOOLS_VERSION - } - - $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" - - Progress ("Downloading Rtools from: " + $rtoolsurl) - bash -c ("curl --silent -o ../Rtools-current.exe -L " + $rtoolsurl) - - Progress "Running Rtools installer" - Start-Process -FilePath ..\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait - - $RtoolsDrive = "C:" - echo "Rtools is now available on drive $RtoolsDrive" - - Progress "Setting PATH" - if ( -not(Test-Path Env:\GCC_PATH) ) { - $gcc_path = "gcc-4.6.3" - } - Else { - $gcc_path = $env:GCC_PATH - } - $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gcc_path + '\bin;' + $env:PATH - $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/' -} - -[CmdletBinding()] -Param() - -Progress "Bootstrap: Start" - -Progress "Adding GnuWin32 tools to PATH" -$env:PATH = "C:\Program Files (x86)\Git\bin;" + $env:PATH - -Progress "Setting time zone" -tzutil /g -tzutil /s "GMT Standard Time" -tzutil /g - -InstallR - -if ((Test-Path "src") -or ($env:USE_RTOOLS)) { - InstallRtools -} -Else { - Progress "Skipping download of Rtools because src/ directory is missing." -} - -$env:PATH.Split(";") - -Progress "Setting R_LIBS_USER" -$env:R_LIBS_USER = 'c:\RLibrary' -if ( -not(Test-Path $env:R_LIBS_USER) ) { - mkdir $env:R_LIBS_USER -} - -Progress "Bootstrap: Done" From f3eb1636d1d9ec6568e8b048d41b5928698d244c Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 17:26:56 +0900 Subject: [PATCH 08/31] Clean up the dependencies installation script --- dev/appveyor-install-dependencies.ps1 | 52 +++++---------------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 339b200a8293..008ec1d1d448 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -17,36 +17,6 @@ under the License. $CRAN = "https://cloud.r-project.org" -# Found at http://zduck.com/2012/powershell-batch-files-exit-codes/ -Function Exec -{ - [CmdletBinding()] - param ( - [Parameter(Position=0, Mandatory=1)] - [scriptblock]$Command, - [Parameter(Position=1, Mandatory=0)] - [string]$ErrorMessage = "Execution of command failed.`n$Command" - ) - $ErrorActionPreference = "Continue" - & $Command 2>&1 | %{ "$_" } - if ($LastExitCode -ne 0) { - throw "Exec: $ErrorMessage`nExit code: $LastExitCode" - } -} - -Function Progress -{ - [CmdletBinding()] - param ( - [Parameter(Position=0, Mandatory=0)] - [string]$Message = "" - ) - - $ProgressMessage = '== ' + (Get-Date) + ': ' + $Message - - Write-Host $ProgressMessage -ForegroundColor Magenta -} - Function InstallR { [CmdletBinding()] Param() @@ -65,8 +35,6 @@ Function InstallR { $arch = $env:R_ARCH } - Progress ("Version: " + $version) - If ($version -eq "devel") { $url_path = "" $version = "devel" @@ -90,29 +58,26 @@ Function InstallR { $url_path = ("old/" + $version + "/") } - Progress ("URL path: " + $url_path) - $rurl = $CRAN + "/bin/windows/base/" + $url_path + "R-" + $version + "-win.exe" - Progress ("Downloading R from: " + $rurl) - Exec { bash -c ("curl --silent -o ../R-win.exe -L " + $rurl) } + # Downloading R + Start-FileDownload $rurl "../R-win.exe" - Progress "Running R installer" + # Running R installer Start-Process -FilePath ..\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait $RDrive = "C:" echo "R is now available on drive $RDrive" - Progress "Setting PATH" $env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH - Progress "Testing R installation" + # Testing R installation Rscript -e "sessionInfo()" } Function InstallRtools { if ( -not(Test-Path Env:\RTOOLS_VERSION) ) { - Progress "Determining Rtools version" + # Determining Rtools version $rtoolsver = $(Invoke-WebRequest ($CRAN + "/bin/windows/Rtools/VERSION.txt")).Content.Split(' ')[2].Split('.')[0..1] -Join '' } Else { @@ -121,16 +86,15 @@ Function InstallRtools { $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" - Progress ("Downloading Rtools from: " + $rtoolsurl) + # Downloading Rtools bash -c ("curl --silent -o ../Rtools-current.exe -L " + $rtoolsurl) - Progress "Running Rtools installer" + # Running Rtools installer Start-Process -FilePath ..\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait $RtoolsDrive = "C:" echo "Rtools is now available on drive $RtoolsDrive" - Progress "Setting PATH" if ( -not(Test-Path Env:\GCC_PATH) ) { $gcc_path = "gcc-4.6.3" } @@ -191,7 +155,7 @@ if ((Test-Path "src") -or ($env:USE_RTOOLS)) { InstallRtools } Else { - Progress "Skipping download of Rtools because src/ directory is missing." + # Skipping download of Rtools because src/ directory is missing. } $env:PATH.Split(";") $env:R_LIBS_USER = 'c:\RLibrary' From fe95491bf7ef28f5ee0d7edb8ec5b14529815bcb Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 17:41:34 +0900 Subject: [PATCH 09/31] Fix comment --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index a5ba9510f418..11af542f3add 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,9 +23,10 @@ configuration: Debug cache: - C:\Users\appveyor\.m2 -# Install maven and dependencies install: + # Install maven and dependencies - ps: .\dev\appveyor-install-dependencies.ps1 + # Required package for R unit tests - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" build_script: From a8e74fc531bb83ceb14d930ca6f03c799fbde384 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 17:43:01 +0900 Subject: [PATCH 10/31] Uppercase for Maven in the comment --- dev/appveyor-install-dependencies.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 008ec1d1d448..8afdeb740b3e 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -113,7 +113,7 @@ if (!(Test-Path $tools)) New-Item -ItemType Directory -Force -Path $tools | Out-Null } -# ========================== maven +# ========================== Maven Push-Location $tools $mavenVer = "3.3.9" From 69fd3f1ca23569db0e1708bde59a0eb7020a7f2c Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 19:12:55 +0900 Subject: [PATCH 11/31] Clean up and make pretty --- dev/appveyor-install-dependencies.ps1 | 55 +++++++++------------------ 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 8afdeb740b3e..2a981fb9e185 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -21,13 +21,6 @@ Function InstallR { [CmdletBinding()] Param() - if ( -not(Test-Path Env:\R_VERSION) ) { - $version = "patched" - } - Else { - $version = $env:R_VERSION - } - if ( -not(Test-Path Env:\R_ARCH) ) { $arch = "i386" } @@ -35,30 +28,30 @@ Function InstallR { $arch = $env:R_ARCH } - If ($version -eq "devel") { + If ($rVer -eq "devel") { $url_path = "" - $version = "devel" + $rVer = "devel" } - ElseIf (($version -eq "stable") -or ($version -eq "release")) { + ElseIf (($rVer -eq "stable") -or ($rVer -eq "release")) { $url_path = "" - $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version - If ($version -eq "3.2.4") { - $version = "3.2.4revised" + $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + If ($rVer -eq "3.2.4") { + $rVer = "3.2.4revised" } } - ElseIf ($version -eq "patched") { + ElseIf ($rVer -eq "patched") { $url_path = "" - $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + "patched" + $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + "patched" } - ElseIf ($version -eq "oldrel") { - $version = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-oldrel).Content).version - $url_path = ("old/" + $version + "/") + ElseIf ($rVer -eq "oldrel") { + $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-oldrel).Content).version + $url_path = ("old/" + $rVer + "/") } Else { - $url_path = ("old/" + $version + "/") + $url_path = ("old/" + $rVer + "/") } - $rurl = $CRAN + "/bin/windows/base/" + $url_path + "R-" + $version + "-win.exe" + $rurl = $CRAN + "/bin/windows/base/" + $url_path + "R-" + $rVer + "-win.exe" # Downloading R Start-FileDownload $rurl "../R-win.exe" @@ -76,14 +69,7 @@ Function InstallR { } Function InstallRtools { - if ( -not(Test-Path Env:\RTOOLS_VERSION) ) { - # Determining Rtools version - $rtoolsver = $(Invoke-WebRequest ($CRAN + "/bin/windows/Rtools/VERSION.txt")).Content.Split(' ')[2].Split('.')[0..1] -Join '' - } - Else { - $rtoolsver = $env:RTOOLS_VERSION - } - + $rtoolsver = rtoolsVer.Split('.')[0..1] -Join '' $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" # Downloading Rtools @@ -149,15 +135,12 @@ $env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" Pop-Location # ========================== R -$env:PATH = "C:\Program Files (x86)\Git\bin;" + $env:PATH +$rVer = "3.3.1" +$rToolsVer = "3.4.0" + InstallR -if ((Test-Path "src") -or ($env:USE_RTOOLS)) { - InstallRtools -} -Else { - # Skipping download of Rtools because src/ directory is missing. -} -$env:PATH.Split(";") +InstallRtools + $env:R_LIBS_USER = 'c:\RLibrary' if ( -not(Test-Path $env:R_LIBS_USER) ) { mkdir $env:R_LIBS_USER From 3a21367e7011b5a381355bd4367b5a52a4c195fb Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 19:15:48 +0900 Subject: [PATCH 12/31] Fix typo in variable names --- dev/appveyor-install-dependencies.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 2a981fb9e185..bf8c61c29fbf 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -69,7 +69,7 @@ Function InstallR { } Function InstallRtools { - $rtoolsver = rtoolsVer.Split('.')[0..1] -Join '' + $rtoolsver = rToolsVer.Split('.')[0..1] -Join '' $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" # Downloading Rtools From 2b9af1549e2302e9603fa733e2099e3be0783a57 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 19:17:50 +0900 Subject: [PATCH 13/31] Fix variable declaration --- dev/appveyor-install-dependencies.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index bf8c61c29fbf..b0b32d0b26c6 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -69,7 +69,7 @@ Function InstallR { } Function InstallRtools { - $rtoolsver = rToolsVer.Split('.')[0..1] -Join '' + $rtoolsver = $rToolsVer.Split('.')[0..1] -Join '' $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" # Downloading Rtools From cdb24a78ce225c126e19f3b6b9b0309581395e84 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 19:26:42 +0900 Subject: [PATCH 14/31] Fix R version to 3.3.0 --- dev/appveyor-install-dependencies.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index b0b32d0b26c6..12c87e0d3e12 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -135,7 +135,7 @@ $env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" Pop-Location # ========================== R -$rVer = "3.3.1" +$rVer = "3.3.0" $rToolsVer = "3.4.0" InstallR From 398830576324eb76d399228ebb484a5896945080 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 20:22:58 +0900 Subject: [PATCH 15/31] Remove meaningless CmdletBinding and Param --- dev/appveyor-install-dependencies.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 12c87e0d3e12..063937a1094a 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -18,9 +18,6 @@ under the License. $CRAN = "https://cloud.r-project.org" Function InstallR { - [CmdletBinding()] - Param() - if ( -not(Test-Path Env:\R_ARCH) ) { $arch = "i386" } From 1f23b0596b98cf333cab303f4b5ab53940bafbca Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 29 Aug 2016 20:41:44 +0900 Subject: [PATCH 16/31] Consistent variable names --- dev/appveyor-install-dependencies.ps1 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 063937a1094a..37d2b05f0f3f 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -26,29 +26,29 @@ Function InstallR { } If ($rVer -eq "devel") { - $url_path = "" + $urlPath = "" $rVer = "devel" } ElseIf (($rVer -eq "stable") -or ($rVer -eq "release")) { - $url_path = "" + $urlPath = "" $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version If ($rVer -eq "3.2.4") { $rVer = "3.2.4revised" } } ElseIf ($rVer -eq "patched") { - $url_path = "" + $urlPath = "" $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + "patched" } ElseIf ($rVer -eq "oldrel") { $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-oldrel).Content).version - $url_path = ("old/" + $rVer + "/") + $urlPath = ("old/" + $rVer + "/") } Else { - $url_path = ("old/" + $rVer + "/") + $urlPath = ("old/" + $rVer + "/") } - $rurl = $CRAN + "/bin/windows/base/" + $url_path + "R-" + $rVer + "-win.exe" + $rurl = $CRAN + "/bin/windows/base/" + $urlPath + "R-" + $rVer + "-win.exe" # Downloading R Start-FileDownload $rurl "../R-win.exe" @@ -79,12 +79,12 @@ Function InstallRtools { echo "Rtools is now available on drive $RtoolsDrive" if ( -not(Test-Path Env:\GCC_PATH) ) { - $gcc_path = "gcc-4.6.3" + $gccPath = "gcc-4.6.3" } Else { - $gcc_path = $env:GCC_PATH + $gccPath = $env:GCC_PATH } - $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gcc_path + '\bin;' + $env:PATH + $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gccPath + '\bin;' + $env:PATH $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/' } From 97f3ea766680f0cf2b73fb77a35bf79c63728bdf Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 30 Aug 2016 13:43:51 +0900 Subject: [PATCH 17/31] Fix R version to 3.3.1 and minimize the codes --- dev/appveyor-install-dependencies.ps1 | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 37d2b05f0f3f..9c1922815de8 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -25,28 +25,11 @@ Function InstallR { $arch = $env:R_ARCH } - If ($rVer -eq "devel") { - $urlPath = "" - $rVer = "devel" - } - ElseIf (($rVer -eq "stable") -or ($rVer -eq "release")) { - $urlPath = "" - $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version - If ($rVer -eq "3.2.4") { - $rVer = "3.2.4revised" - } - } - ElseIf ($rVer -eq "patched") { - $urlPath = "" - $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + "patched" - } - ElseIf ($rVer -eq "oldrel") { - $rVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-oldrel).Content).version + $urlPath = "" + $latestVer = $(ConvertFrom-JSON $(Invoke-WebRequest http://rversions.r-pkg.org/r-release).Content).version + If ($rVer -ne $latestVer) { $urlPath = ("old/" + $rVer + "/") } - Else { - $urlPath = ("old/" + $rVer + "/") - } $rurl = $CRAN + "/bin/windows/base/" + $urlPath + "R-" + $rVer + "-win.exe" @@ -132,7 +115,7 @@ $env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" Pop-Location # ========================== R -$rVer = "3.3.0" +$rVer = "3.3.1" $rToolsVer = "3.4.0" InstallR From e7addc938691c2da420c2fef2e7b9131fdec00b2 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 30 Aug 2016 13:53:24 +0900 Subject: [PATCH 18/31] Consistent downloading via Start-FileDownload --- dev/appveyor-install-dependencies.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 9c1922815de8..849b5be8b056 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -53,7 +53,7 @@ Function InstallRtools { $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" # Downloading Rtools - bash -c ("curl --silent -o ../Rtools-current.exe -L " + $rtoolsurl) + Start-FileDownload $rtoolsurl "../Rtools-current.exe" # Running Rtools installer Start-Process -FilePath ..\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait From 1b7b5f31e3eb833675045cad79aa2e73e77adc21 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 30 Aug 2016 14:27:31 +0900 Subject: [PATCH 19/31] Fix styles and nits --- dev/appveyor-install-dependencies.ps1 | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index 849b5be8b056..087b8666cc68 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -34,10 +34,10 @@ Function InstallR { $rurl = $CRAN + "/bin/windows/base/" + $urlPath + "R-" + $rVer + "-win.exe" # Downloading R - Start-FileDownload $rurl "../R-win.exe" + Start-FileDownload $rurl "R-win.exe" # Running R installer - Start-Process -FilePath ..\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait + Start-Process -FilePath .\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait $RDrive = "C:" echo "R is now available on drive $RDrive" @@ -53,10 +53,10 @@ Function InstallRtools { $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe" # Downloading Rtools - Start-FileDownload $rtoolsurl "../Rtools-current.exe" + Start-FileDownload $rtoolsurl "Rtools-current.exe" # Running Rtools installer - Start-Process -FilePath ..\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait + Start-Process -FilePath .\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait $RtoolsDrive = "C:" echo "Rtools is now available on drive $RtoolsDrive" @@ -74,8 +74,7 @@ Function InstallRtools { # create tools directory outside of Spark directory $up = (Get-Item -Path ".." -Verbose).FullName $tools = "$up\tools" -if (!(Test-Path $tools)) -{ +if (!(Test-Path $tools)) { New-Item -ItemType Directory -Force -Path $tools | Out-Null } @@ -98,8 +97,7 @@ Pop-Location # ========================== Hadoop bin package $hadoopVer = "2.6.0" $hadoopPath = "$tools\hadoop" -if (!(Test-Path $hadoopPath)) -{ +if (!(Test-Path $hadoopPath)) { New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null } Push-Location $hadoopPath From b1a50764dcc71981fdc96e5a4b8d2e208f7692ec Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 31 Aug 2016 15:42:18 +0900 Subject: [PATCH 20/31] Tests with hive related ones as well --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 11af542f3add..0390c253e670 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,7 +30,7 @@ install: - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" build_script: - - cmd: mvn -DskipTests -Psparkr package + - cmd: mvn -DskipTests -Psparkr -Phive -Phive-thriftserver package test_script: - cmd: .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///" R\pkg\tests\run-all.R From 2cc7a4729ff691970c534c772264d130a836abe6 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 13:10:19 +0900 Subject: [PATCH 21/31] Add a guide and file change detection for R --- appveyor.yml | 4 ++ dev/appveyor-guide.md | 128 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 dev/appveyor-guide.md diff --git a/appveyor.yml b/appveyor.yml index 0390c253e670..92d890c625fe 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,6 +20,10 @@ shallow_clone: true platform: x64 configuration: Debug +only_commits: + files: + - R/ + cache: - C:\Users\appveyor\.m2 diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md new file mode 100644 index 000000000000..0e6452b2d1df --- /dev/null +++ b/dev/appveyor-guide.md @@ -0,0 +1,128 @@ +# Appveyor Guides + +Currently, SparkR on Windows is being tested with [Appveyor](https://ci.appveyor.com). This page describes how to set up Appveyor with Spark, how to run the build, check the status and stop the build via this tool. There is the documenation for Appveyor [here](https://www.appveyor.com/docs). Please refer this for full details. + + +### Setting up Appveyor + +##### Sign up Appveyor. + +2016-09-04 11 07 48 + +Go to https://ci.appveyor.com, and then click "SIGN UP FOR FREE". + +2016-09-04 11 07 58 + +As Apache Spark is one of open source projects, click "FREE - for open-source projects". + +2016-09-04 11 08 10 + +Click "Github". + + +##### After signing up, go to profile to link Github and Appveyor. + +2016-09-04 11 09 43 + +Click your account and then click "Profile". + +2016-09-04 11 09 52 + +Enable the link with GitHub via clicking "Link Github account". + +2016-09-04 11 10 05 + +Click "Authorize application" in Github site. + + +##### Add a project, Spark to enable the builds. + +2016-08-30 12 16 31 + +Go to the PROJECTS menu. + +2016-08-30 12 16 35 + +Click "NEW PROJECT" to add Spark. + +2016-09-04 11 10 22 + +Since we will use Github here, click the "GITHUB" button and then click "Authorize Github" so that Appveyor can access to the Github logs (e.g. commits). + +2016-09-04 11 10 27 + +Click "Authorize application" from Github (the above step will pop up this page.) + +2016-09-04 11 10 36 + +Come back to https://ci.appveyor.com/projects/new and then adds "spark". + + +##### Check if any event supposed to run the build actually triggers the build. + +2016-08-30 12 16 31 + +Click "PROJECTS" menu. + +2016-09-04 11 22 37 + +Click Spark project. + + + +### Checking the status, restarting and stopping the build + +2016-08-30 12 16 31 + +Click "PROJECTS" menu. + +2016-09-04 11 22 37 + +Locate "spark" and click it. + +2016-09-04 11 23 24 + +Here, we can check the status of current build. Also, "HISTORY" shows the past build history. + +2016-08-30 12 29 41 + +If the build is stopped, "RE-BUILD COMMIT" button appears. Click this button to restart the build. + +2016-08-30 1 11 13 + +If the build is running, "CANCEL BUILD" buttom appears. Click this button top cancel the current build. + + +### Specifying the branch for building and setting the build schedule + +2016-08-30 1 19 12 + +Click the settings button on the right. + +2016-08-30 12 42 25 + +Set the default branch to build as above. + +2016-08-30 12 42 33 + +Specify the branch in order to exclude the builds in other branches. + +2016-08-30 12 42 43 + +Set the Crontab expression to regularly start the build. Appveyor uses Crontab expression, [atifaziz/NCrontab](https://github.com/atifaziz/NCrontab/wiki/Crontab-Expression). Please refer the examples [here](https://github.com/atifaziz/NCrontab/wiki/Crontab-Examples). + + + +### Filtering commits and Pull Requests + +Currently, Appveyor is only used for SparkR. So, the build is only triggered when R codes are changed. + +This is specified in `.appveyor.yml` as below: + +``` +only_commits: + files: + - R/ +``` + +Please refer https://www.appveyor.com/docs/how-to/filtering-commits for more details. From 4f2db1e931753908784530b356569b83514ba5af Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 13:11:43 +0900 Subject: [PATCH 22/31] Make the guide pretty --- dev/appveyor-guide.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md index 0e6452b2d1df..c9ff38b907d2 100644 --- a/dev/appveyor-guide.md +++ b/dev/appveyor-guide.md @@ -5,7 +5,7 @@ Currently, SparkR on Windows is being tested with [Appveyor](https://ci.appveyor ### Setting up Appveyor -##### Sign up Appveyor. +#### Sign up Appveyor. 2016-09-04 11 07 48 @@ -20,7 +20,7 @@ As Apache Spark is one of open source projects, click "FREE - for open-source pr Click "Github". -##### After signing up, go to profile to link Github and Appveyor. +#### After signing up, go to profile to link Github and Appveyor. 2016-09-04 11 09 43 @@ -35,7 +35,7 @@ Enable the link with GitHub via clicking "Link Github account". Click "Authorize application" in Github site. -##### Add a project, Spark to enable the builds. +#### Add a project, Spark to enable the builds. 2016-08-30 12 16 31 @@ -58,7 +58,7 @@ Click "Authorize application" from Github (the above step will pop up this page. Come back to https://ci.appveyor.com/projects/new and then adds "spark". -##### Check if any event supposed to run the build actually triggers the build. +#### Check if any event supposed to run the build actually triggers the build. 2016-08-30 12 16 31 From 2e049111d7cdbc1c11fd1495a3e438d6cd2fd4c7 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 13:18:09 +0900 Subject: [PATCH 23/31] Make the guide better --- dev/appveyor-guide.md | 102 +++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md index c9ff38b907d2..8b2519bc9863 100644 --- a/dev/appveyor-guide.md +++ b/dev/appveyor-guide.md @@ -6,110 +6,112 @@ Currently, SparkR on Windows is being tested with [Appveyor](https://ci.appveyor ### Setting up Appveyor #### Sign up Appveyor. - -2016-09-04 11 07 48 -Go to https://ci.appveyor.com, and then click "SIGN UP FOR FREE". +- Go to https://ci.appveyor.com, and then click "SIGN UP FOR FREE". -2016-09-04 11 07 58 + 2016-09-04 11 07 48 -As Apache Spark is one of open source projects, click "FREE - for open-source projects". +- As Apache Spark is one of open source projects, click "FREE - for open-source projects". + + 2016-09-04 11 07 58 -2016-09-04 11 08 10 +- Click "Github". -Click "Github". + 2016-09-04 11 08 10 #### After signing up, go to profile to link Github and Appveyor. -2016-09-04 11 09 43 +- Click your account and then click "Profile". -Click your account and then click "Profile". + 2016-09-04 11 09 43 -2016-09-04 11 09 52 +- Enable the link with GitHub via clicking "Link Github account". -Enable the link with GitHub via clicking "Link Github account". + 2016-09-04 11 09 52 -2016-09-04 11 10 05 +- Click "Authorize application" in Github site. -Click "Authorize application" in Github site. +2016-09-04 11 10 05 #### Add a project, Spark to enable the builds. -2016-08-30 12 16 31 +- Go to the PROJECTS menu. -Go to the PROJECTS menu. - -2016-08-30 12 16 35 - -Click "NEW PROJECT" to add Spark. - -2016-09-04 11 10 22 + 2016-08-30 12 16 31 -Since we will use Github here, click the "GITHUB" button and then click "Authorize Github" so that Appveyor can access to the Github logs (e.g. commits). +- Click "NEW PROJECT" to add Spark. + + 2016-08-30 12 16 35 -2016-09-04 11 10 27 +- Since we will use Github here, click the "GITHUB" button and then click "Authorize Github" so that Appveyor can access to the Github logs (e.g. commits). -Click "Authorize application" from Github (the above step will pop up this page.) - -2016-09-04 11 10 36 + 2016-09-04 11 10 22 + +- Click "Authorize application" from Github (the above step will pop up this page). + + 2016-09-04 11 10 27 + +- Come back to https://ci.appveyor.com/projects/new and then adds "spark". -Come back to https://ci.appveyor.com/projects/new and then adds "spark". + 2016-09-04 11 10 36 #### Check if any event supposed to run the build actually triggers the build. -2016-08-30 12 16 31 +- Click "PROJECTS" menu. -Click "PROJECTS" menu. + 2016-08-30 12 16 31 -2016-09-04 11 22 37 +- Click Spark project. -Click Spark project. + 2016-09-04 11 22 37 ### Checking the status, restarting and stopping the build -2016-08-30 12 16 31 +- Click "PROJECTS" menu. -Click "PROJECTS" menu. + 2016-08-30 12 16 31 -2016-09-04 11 22 37 +- Locate "spark" and click it. -Locate "spark" and click it. + 2016-09-04 11 22 37 -2016-09-04 11 23 24 +- Here, we can check the status of current build. Also, "HISTORY" shows the past build history. -Here, we can check the status of current build. Also, "HISTORY" shows the past build history. + 2016-09-04 11 23 24 -2016-08-30 12 29 41 +- If the build is stopped, "RE-BUILD COMMIT" button appears. Click this button to restart the build. -If the build is stopped, "RE-BUILD COMMIT" button appears. Click this button to restart the build. + 2016-08-30 12 29 41 -2016-08-30 1 11 13 +- If the build is running, "CANCEL BUILD" buttom appears. Click this button top cancel the current build. -If the build is running, "CANCEL BUILD" buttom appears. Click this button top cancel the current build. + 2016-08-30 1 11 13 ### Specifying the branch for building and setting the build schedule -2016-08-30 1 19 12 +- Click the settings button on the right. + + 2016-08-30 1 19 12 + +- Set the default branch to build as above. + + 2016-08-30 12 42 25 -Click the settings button on the right. - -2016-08-30 12 42 25 +- Specify the branch in order to exclude the builds in other branches. -Set the default branch to build as above. + 2016-08-30 12 42 33 -2016-08-30 12 42 33 +- Set the Crontab expression to regularly start the build. Appveyor uses Crontab expression, [atifaziz/NCrontab](https://github.com/atifaziz/NCrontab/wiki/Crontab-Expression). Please refer the examples [here](https://github.com/atifaziz/NCrontab/wiki/Crontab-Examples). -Specify the branch in order to exclude the builds in other branches. -2016-08-30 12 42 43 + 2016-08-30 12 42 43 -Set the Crontab expression to regularly start the build. Appveyor uses Crontab expression, [atifaziz/NCrontab](https://github.com/atifaziz/NCrontab/wiki/Crontab-Expression). Please refer the examples [here](https://github.com/atifaziz/NCrontab/wiki/Crontab-Examples). From 43a5a44237d70261369e0d6d26ef12cbde604a3d Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 15:38:55 +0900 Subject: [PATCH 24/31] Build against master only --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 92d890c625fe..8950826101fc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,6 +20,10 @@ shallow_clone: true platform: x64 configuration: Debug +branches: + only: + - master + only_commits: files: - R/ From e6793a74a7a931c1f84d3ba8b96076dd27ba0021 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 16:04:40 +0900 Subject: [PATCH 25/31] Add the documentation for UI configuration and appveyor.yml --- dev/appveyor-guide.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md index 8b2519bc9863..0b80be589ba8 100644 --- a/dev/appveyor-guide.md +++ b/dev/appveyor-guide.md @@ -95,6 +95,9 @@ Currently, SparkR on Windows is being tested with [Appveyor](https://ci.appveyor ### Specifying the branch for building and setting the build schedule +Note: It seems Web UI configurations and `appveyor.yml` are mutually exclusive according to the [documentation](https://www.appveyor.com/docs/build-configuration/#configuring-build). + + - Click the settings button on the right. 2016-08-30 1 19 12 From ceef9bf03b286910b5b96aa6129f537c8c7cae54 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 16:09:54 +0900 Subject: [PATCH 26/31] Change Appveyor to AppVeyor --- dev/appveyor-guide.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md index 0b80be589ba8..0fef42e59251 100644 --- a/dev/appveyor-guide.md +++ b/dev/appveyor-guide.md @@ -1,11 +1,11 @@ -# Appveyor Guides +# AppVeyor Guides -Currently, SparkR on Windows is being tested with [Appveyor](https://ci.appveyor.com). This page describes how to set up Appveyor with Spark, how to run the build, check the status and stop the build via this tool. There is the documenation for Appveyor [here](https://www.appveyor.com/docs). Please refer this for full details. +Currently, SparkR on Windows is being tested with [AppVeyor](https://ci.appveyor.com). This page describes how to set up AppVeyor with Spark, how to run the build, check the status and stop the build via this tool. There is the documenation for AppVeyor [here](https://www.appveyor.com/docs). Please refer this for full details. -### Setting up Appveyor +### Setting up AppVeyor -#### Sign up Appveyor. +#### Sign up AppVeyor. - Go to https://ci.appveyor.com, and then click "SIGN UP FOR FREE". @@ -20,7 +20,7 @@ Currently, SparkR on Windows is being tested with [Appveyor](https://ci.appveyor 2016-09-04 11 08 10 -#### After signing up, go to profile to link Github and Appveyor. +#### After signing up, go to profile to link Github and AppVeyor. - Click your account and then click "Profile". @@ -45,7 +45,7 @@ Currently, SparkR on Windows is being tested with [Appveyor](https://ci.appveyor 2016-08-30 12 16 35 -- Since we will use Github here, click the "GITHUB" button and then click "Authorize Github" so that Appveyor can access to the Github logs (e.g. commits). +- Since we will use Github here, click the "GITHUB" button and then click "Authorize Github" so that AppVeyor can access to the Github logs (e.g. commits). 2016-09-04 11 10 22 @@ -110,7 +110,7 @@ Note: It seems Web UI configurations and `appveyor.yml` are mutually exclusive 2016-08-30 12 42 33 -- Set the Crontab expression to regularly start the build. Appveyor uses Crontab expression, [atifaziz/NCrontab](https://github.com/atifaziz/NCrontab/wiki/Crontab-Expression). Please refer the examples [here](https://github.com/atifaziz/NCrontab/wiki/Crontab-Examples). +- Set the Crontab expression to regularly start the build. AppVeyor uses Crontab expression, [atifaziz/NCrontab](https://github.com/atifaziz/NCrontab/wiki/Crontab-Expression). Please refer the examples [here](https://github.com/atifaziz/NCrontab/wiki/Crontab-Examples). 2016-08-30 12 42 43 @@ -120,7 +120,7 @@ Note: It seems Web UI configurations and `appveyor.yml` are mutually exclusive ### Filtering commits and Pull Requests -Currently, Appveyor is only used for SparkR. So, the build is only triggered when R codes are changed. +Currently, AppVeyor is only used for SparkR. So, the build is only triggered when R codes are changed. This is specified in `.appveyor.yml` as below: From ccf176da8b6b8742e18abdf8058ec0d866956de5 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 4 Sep 2016 20:28:15 +0900 Subject: [PATCH 27/31] Fix typo in the guide line --- dev/appveyor-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md index 0fef42e59251..ed51fdf18202 100644 --- a/dev/appveyor-guide.md +++ b/dev/appveyor-guide.md @@ -95,7 +95,7 @@ Currently, SparkR on Windows is being tested with [AppVeyor](https://ci.appveyor ### Specifying the branch for building and setting the build schedule -Note: It seems Web UI configurations and `appveyor.yml` are mutually exclusive according to the [documentation](https://www.appveyor.com/docs/build-configuration/#configuring-build). +Note: It seems the configurations in UI and `appveyor.yml` are mutually exclusive according to the [documentation](https://www.appveyor.com/docs/build-configuration/#configuring-build). - Click the settings button on the right. From 8e1954b15ec9fb64e010f30a52afd90260c01df7 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 8 Sep 2016 11:33:44 +0900 Subject: [PATCH 28/31] Add -Phadoop-2.6 profile --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 8950826101fc..f8cc5e6fb518 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,7 +38,7 @@ install: - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" build_script: - - cmd: mvn -DskipTests -Psparkr -Phive -Phive-thriftserver package + - cmd: mvn -DskipTests -Phadoop-2.6 -Psparkr -Phive -Phive-thriftserver package test_script: - cmd: .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///" R\pkg\tests\run-all.R From 41dbdcfb22ba2748611ef9b0d04be5caeb0f6db9 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 8 Sep 2016 11:47:04 +0900 Subject: [PATCH 29/31] Add the documenation for checking full details of failed tests --- dev/appveyor-guide.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md index ed51fdf18202..0ee51d178b8b 100644 --- a/dev/appveyor-guide.md +++ b/dev/appveyor-guide.md @@ -131,3 +131,41 @@ only_commits: ``` Please refer https://www.appveyor.com/docs/how-to/filtering-commits for more details. + + +### Checking the full log of the build + +Currently, the console in AppVeyor does not print full details. This can be manually checked. For example, AppVeyor shows the failed tests as below in console + +``` +Failed ------------------------------------------------------------------------- +1. Error: union on two RDDs (@test_binary_function.R#38) ----------------------- +1: textFile(sc, fileName) at C:/projects/spark/R/lib/SparkR/tests/testthat/test_binary_function.R:38 +2: callJMethod(sc, "textFile", path, getMinPartitions(sc, minPartitions)) +3: invokeJava(isStatic = FALSE, objId$id, methodName, ...) +4: stop(readString(conn)) +``` + +After downloading the log by clicking the log button as below: + +![2016-09-08 11 37 17](https://cloud.githubusercontent.com/assets/6477701/18335227/b07d0782-75b8-11e6-94da-1b88cd2a2402.png) + +the details can be checked as below (e.g. exceptions) + +``` +Failed ------------------------------------------------------------------------- +1. Error: spark.lda with text input (@test_mllib.R#655) ------------------------ + org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:/projects/spark/R/lib/SparkR/tests/testthat/data/mllib/sample_lda_data.txt; + at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$12.apply(DataSource.scala:376) + at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$12.apply(DataSource.scala:365) + at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) + at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) + ... + + 1: read.text("data/mllib/sample_lda_data.txt") at C:/projects/spark/R/lib/SparkR/tests/testthat/test_mllib.R:655 + 2: dispatchFunc("read.text(path)", x, ...) + 3: f(x, ...) + 4: callJMethod(read, "text", paths) + 5: invokeJava(isStatic = FALSE, objId$id, methodName, ...) + 6: stop(readString(conn)) +``` From fe9419d65374d944f158a9ee3f3fd1979791bbc5 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 8 Sep 2016 11:48:59 +0900 Subject: [PATCH 30/31] Consistent newlines for each title --- dev/appveyor-guide.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/appveyor-guide.md b/dev/appveyor-guide.md index 0ee51d178b8b..d2e00b484727 100644 --- a/dev/appveyor-guide.md +++ b/dev/appveyor-guide.md @@ -69,7 +69,6 @@ Currently, SparkR on Windows is being tested with [AppVeyor](https://ci.appveyor 2016-09-04 11 22 37 - ### Checking the status, restarting and stopping the build - Click "PROJECTS" menu. @@ -116,8 +115,6 @@ Note: It seems the configurations in UI and `appveyor.yml` are mutually exclusi 2016-08-30 12 42 43 - - ### Filtering commits and Pull Requests Currently, AppVeyor is only used for SparkR. So, the build is only triggered when R codes are changed. From 9c06b92f7ec01442c5ba0076ccf32ba91b847b3d Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 8 Sep 2016 15:34:58 +0900 Subject: [PATCH 31/31] Adds survival and e1071 for dependencies --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index f8cc5e6fb518..5e756835bcb9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -36,6 +36,11 @@ install: - ps: .\dev\appveyor-install-dependencies.ps1 # Required package for R unit tests - cmd: R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" + - cmd: R -e "packageVersion('testthat')" + - cmd: R -e "install.packages('e1071', repos='http://cran.us.r-project.org')" + - cmd: R -e "packageVersion('e1071')" + - cmd: R -e "install.packages('survival', repos='http://cran.us.r-project.org')" + - cmd: R -e "packageVersion('survival')" build_script: - cmd: mvn -DskipTests -Phadoop-2.6 -Psparkr -Phive -Phive-thriftserver package