Spark configuration and tests from parsnip

.github/workflows/CRAN-R-CMD-check.yaml

@@ -1,7 +1,13 @@
 on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
   schedule:
-    # runs tests every day at 1am
-    - cron:  '0 1 * * *'
+    # runs tests every day at 1am EST
+    - cron:  '0 5 * * *'
 
 name: CRAN-R-CMD-check
 
@@ -15,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {os: macOS-latest,   r: 'devel'}
+          - {os: ubuntu-16.04,   r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
           - {os: windows-latest, r: 'release'}
 
     env:
@@ -32,6 +38,12 @@ jobs:
 
       - uses: r-lib/actions/setup-pandoc@master
 
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '8' # The JDK version to make available on the path.
+          java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+          architecture: x64 # (x64 or x86) - defaults to x64
+
       - name: Query dependencies
         run: |
           install.packages('remotes')
@@ -52,9 +64,11 @@ jobs:
         env:
           RHUB_PLATFORM: linux-x86_64-ubuntu-gcc
         run: |
-          Rscript -e "remotes::install_github('r-hub/sysreqs')"
-          sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
-          sudo -s eval "$sysreqs"
+          while read -r cmd
+          do
+            eval sudo $cmd
+          done < <(Rscript -e 'cat(remotes::system_requirements("ubuntu", "16.04"), sep = "\n")')
+
 
       - name: Install dependencies
         run: |
@@ -82,7 +96,7 @@ jobs:
 
       - name: Upload check results
         if: failure()
-        uses: actions/upload-artifact@master
+        uses: actions/upload-artifact@main
         with:
           name: ${{ runner.os }}-r${{ matrix.config.r }}-results
           path: check

.github/workflows/GH-R-CMD-check.yaml

@@ -1,7 +1,13 @@
 on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
   schedule:
-    # runs tests every day at 1am
-    - cron:  '0 1 * * *'
+    # runs tests every day at 1am EST
+    - cron:  '0 5 * * *'
 
 name: GH-R-CMD-check
 
@@ -15,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {os: macOS-latest,   r: 'devel'}
+          - {os: ubuntu-16.04,   r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
           - {os: windows-latest, r: 'release'}
 
     env:
@@ -32,6 +38,12 @@ jobs:
 
       - uses: r-lib/actions/setup-pandoc@master
 
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '8' # The JDK version to make available on the path.
+          java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+          architecture: x64 # (x64 or x86) - defaults to x64
+
       - name: Query dependencies
         run: |
           install.packages('remotes')
@@ -52,9 +64,11 @@ jobs:
         env:
           RHUB_PLATFORM: linux-x86_64-ubuntu-gcc
         run: |
-          Rscript -e "remotes::install_github('r-hub/sysreqs')"
-          sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
-          sudo -s eval "$sysreqs"
+          while read -r cmd
+          do
+            eval sudo $cmd
+          done < <(Rscript -e 'cat(remotes::system_requirements("ubuntu", "16.04"), sep = "\n")')
+
 
       - name: Install dependencies
         run: |
@@ -76,7 +90,6 @@ jobs:
           try(remotes::install_github("tidymodels/modeldata"), silent = TRUE)
         shell: Rscript {0}
 
-
       - name: Session info
         run: |
           options(width = 100)
@@ -97,7 +110,7 @@ jobs:
 
       - name: Upload check results
         if: failure()
-        uses: actions/upload-artifact@master
+        uses: actions/upload-artifact@main
         with:
           name: ${{ runner.os }}-r${{ matrix.config.r }}-results
           path: check

.github/workflows/pr-commands.yaml

@@ -21,6 +21,8 @@ jobs:
         run: Rscript -e 'roxygen2::roxygenise()'
       - name: commit
         run: |
+          git config --local user.email "actions@github.com"
+          git config --local user.name "GitHub Actions"
           git add man/\* NAMESPACE
           git commit -m 'Document'
       - uses: r-lib/actions/pr-push@master
@@ -44,6 +46,8 @@ jobs:
         run: Rscript -e 'styler::style_pkg()'
       - name: commit
         run: |
+          git config --local user.email "actions@github.com"
+          git config --local user.name "GitHub Actions"
           git add \*.R
           git commit -m 'Style'
       - uses: r-lib/actions/pr-push@master

DESCRIPTION

@@ -21,9 +21,17 @@ Suggests:
     ranger,
     randomForest,
     C50,
-    earth
+    earth,
+    sparklyr,
+    dplyr,
+    glmnet,
+    rstanarm,
+    modeldata,
+    parsnip,
+    purrr,
+    tidyr,
+    tibble,
+    rlang
 Language: en-US
 Depends: 
-    tidymodels,
-    modeldata,
-    parsnip
+    tidymodels

inst/WORDLIST

@@ -1,3 +1,4 @@
 Lifecycle
 tidymodels
 cron
+GH

tests/testthat.R

@@ -1,4 +1,48 @@
 library(testthat)
 library(extratests)
 
-test_check("extratests")
+## -----------------------------------------------------------------------------
+
+spark_install_winutils <- function(version) {
+  hadoop_version <- if (version < "2.0.0") "2.6" else "2.7"
+  spark_dir <- paste("spark-", version, "-bin-hadoop", hadoop_version, sep = "")
+  winutils_dir <- file.path(Sys.getenv("LOCALAPPDATA"), "spark", spark_dir, "tmp", "hadoop", "bin", fsep = "\\")
+
+  if (!dir.exists(winutils_dir)) {
+    message("Installing winutils...")
+
+    dir.create(winutils_dir, recursive = TRUE)
+    winutils_path <- file.path(winutils_dir, "winutils.exe", fsep = "\\")
+
+    download.file(
+      "https://github.com/steveloughran/winutils/raw/master/hadoop-2.6.0/bin/winutils.exe",
+      winutils_path,
+      mode = "wb"
+    )
+
+    message("Installed winutils in ", winutils_path)
+  }
+}
+
+## -----------------------------------------------------------------------------
+
+library(sparklyr)
+
+if (.Platform$OS.type == "windows") {
+  # Right now, this does not seem to be working on windows; working on fixing it.
+  # Leaving it as-is as a place to start.
+  spark_install_winutils("2.4")
+  sparklyr::spark_install(verbose = TRUE, version = "2.4", hadoop_version = "2.7")
+} else {
+  sparklyr::spark_install(verbose = TRUE, version = "2.4")
+}
+
+sc <- try(sparklyr::spark_connect(master = "local"), silent = TRUE)
+
+if(inherits(sc, "try-error")) {
+  print(sc)
+}
+
+## -----------------------------------------------------------------------------
+
+test_check("extratests", reporter = "summary")

tests/testthat/parsnip-helper-objects.R

@@ -0,0 +1,16 @@
+library(modeldata)
+library(parsnip)
+
+## -----------------------------------------------------------------------------
+
+data("wa_churn")
+data("lending_club")
+data("hpc_data")
+
+# ------------------------------------------------------------------------------
+
+ctrl          <- control_parsnip(verbosity = 1, catch = FALSE)
+caught_ctrl   <- control_parsnip(verbosity = 1, catch = TRUE)
+quiet_ctrl    <- control_parsnip(verbosity = 0, catch = TRUE)
+
+run_glmnet <- utils::compareVersion('3.6.0', as.character(getRversion())) > 0

tests/testthat/test-aaa-spark-install.R

@@ -0,0 +1,5 @@
+context("spark installation")
+
+test_that('is spark installed?', {
+  expect_true(sparklyr::spark_install_find()$installed)
+})

tests/testthat/test-data-descriptors-spark.R

@@ -0,0 +1,84 @@
+library(testthat)
+library(parsnip)
+
+context("spark descriptors")
+
+source(test_path("parsnip-helper-objects.R"))
+hpc <- hpc_data[1:150, c(2:5, 8)] %>% as.data.frame()
+
+# ------------------------------------------------------------------------------
+
+context("descriptor variables")
+
+# ------------------------------------------------------------------------------
+
+template <- function(col, pred, ob, lev, fact, dat, x, y) {
+  lst <- list(.cols = col, .preds = pred, .obs = ob,
+              .lvls = lev, .facts = fact, .dat = dat,
+              .x = x, .y = y)
+
+  Filter(Negate(is.null), lst)
+}
+
+eval_descrs <- function(descrs, not = NULL) {
+
+  if (!is.null(not)) {
+    for (descr in not) {
+      descrs[[descr]] <- NULL
+    }
+  }
+
+  lapply(descrs, do.call, list())
+}
+
+class_tab <- table(hpc$class, dnn = NULL)
+
+# ------------------------------------------------------------------------------
+
+
+
+test_that("spark descriptor", {
+
+  skip_if_not_installed("sparklyr")
+
+  library(sparklyr)
+  library(dplyr)
+
+  sc <- try(spark_connect(master = "local"), silent = TRUE)
+
+  skip_if(inherits(sc, "try-error"))
+
+  npk_descr  <- copy_to(sc,  npk[, 1:4],  "npk_descr", overwrite = TRUE)
+  hpc_descr <- copy_to(sc,        hpc, "hpc_descr", overwrite = TRUE)
+
+  # spark does not allow .x, .y, .dat; spark handles factors differently
+  template2 <- purrr::partial(template, x = NULL, y = NULL, dat = NULL)
+  eval_descrs2 <- purrr::partial(eval_descrs, not = c(".x", ".y", ".dat"))
+  class_tab2 <- table(as.character(hpc$class), dnn = NULL)
+
+  expect_equal(
+    template2(6, 4, 150, NA, 1),
+    eval_descrs2(parsnip:::get_descr_form(compounds ~ ., data = hpc_descr))
+  )
+  expect_equal(
+    template2(3, 1, 150, NA, 1),
+    eval_descrs2(parsnip:::get_descr_form(compounds ~ class, data = hpc_descr))
+  )
+  expect_equal(
+    template2(1, 1, 150, NA, 0),
+    eval_descrs2(parsnip:::get_descr_form(compounds ~ input_fields, data = hpc_descr))
+  )
+  expect_equivalent(
+    template2(4, 4, 150, class_tab2, 0),
+    eval_descrs2(parsnip:::get_descr_form(class ~ ., data = hpc_descr))
+  )
+  expect_equal(
+    template2(1, 1, 150, class_tab2, 0),
+    eval_descrs2(parsnip:::get_descr_form(class ~ input_fields, data = hpc_descr))
+  )
+  expect_equivalent(
+    template2(7, 3, 24, rev(table(npk$K, dnn = NULL)), 3),
+    eval_descrs2(parsnip:::get_descr_form(K ~ ., data = npk_descr))
+  )
+
+})