apacheGH-41402: [CI][R] Update our backwards compatibility CI any oth…

…er R 4.4 cleanups (apache#41403) ### Rationale for this change Keep up with the state of the world, ensure we are maintaining backwards compatibility. Resolves apache#41402 ### What changes are included in this PR? * Bump to 4.4 as the release * Remove old 3.6 jobs now that we no longer support that; clean up code where we hardcode things fro 3.6 and below * Move many of our CI jobs to [rhub's new containers](https://github.com/r-hub/containers). We were accidentally running stale R devel (from December 2023) because the other rhub images stopped being updated. (One exception to be done as a follow on: apache#41416) * Resolve a number of extended test failures With this PR R extended tests should be all green with the exceptions of: * Two sanitizer jobs (test-fedora-r-clang-sanitizer, test-ubuntu-r-sanitizer) — which are being investigated / fixed in apache#41421 * Valgrind — I'm running one last run with a new suppression file. * Binary jobs — these work but fail at upload, see apache#41403 (comment) * Windows R Release — failing on main, apache#41398 ### Are these changes tested? By definition. ### Are there any user-facing changes? No. * GitHub Issue: apache#41402 Lead-authored-by: Jonathan Keane <jkeane@gmail.com> Co-authored-by: Jacob Wujciak-Jens <jacob@wujciak.de> Signed-off-by: Jonathan Keane <jkeane@gmail.com>
vibhatha · May 25, 2024 · ebc9be0 · ebc9be0
1 parent 8fa38be
commit ebc9be0
Show file tree

Hide file tree

Showing 31 changed files with 139 additions and 164 deletions.
diff --git a/.env b/.env
@@ -71,12 +71,12 @@ NUMBA=latest
 NUMPY=latest
 PANDAS=latest
 PYTHON=3.8
-R=4.2
+R=4.4
 SPARK=master
 TURBODBC=latest
 
-# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
-R_IMAGE=ubuntu-gcc-release
+# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-release:latest
+R_IMAGE=ubuntu-release
 R_ORG=rhub
 R_TAG=latest
 

diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -121,7 +121,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- r: ["4.3"]
+ r: ["4.4"]
  ubuntu: [20.04]
  force-tests: ["true"]
  env:
@@ -192,7 +192,7 @@ jobs:
  fail-fast: false
  matrix:
  config:
- - { org: "rhub", image: "debian-gcc-devel", tag: "latest", devtoolset: "" }
+ - { org: "rhub", image: "ubuntu-gcc12", tag: "latest", devtoolset: "" }
  env:
  R_ORG: ${{ matrix.config.org }}
  R_IMAGE: ${{ matrix.config.image }}

diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile
@@ -18,7 +18,7 @@
 ARG base
 FROM ${base}
 
-ARG r=4.2
+ARG r=4.4
 ARG jdk=8
 
 # See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/

diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile
@@ -40,7 +40,7 @@ RUN apt-get update && \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
-ARG r=4.2
+ARG r=4.4
 RUN wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \
  tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \
  # NOTE: Only R >= 4.0 is available in this repo

diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile
@@ -35,7 +35,7 @@ ENV LANG=C.UTF-8
 # Build R
 # [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04
 # [2] https://linuxize.com/post/how-to-install-r-on-ubuntu-18-04/#installing-r-packages-from-cran
-ARG r=3.6
+ARG r=4.4
 RUN apt-get update -y && \
  apt-get install -y \
  dirmngr \

diff --git a/ci/etc/valgrind-cran.supp b/ci/etc/valgrind-cran.supp
@@ -16,7 +16,7 @@
 # under the License.
 
 {
- # `testthat::skip()`s cause a valgrind error that does not show up on CRAN. 
+ # `testthat::skip()`s cause a valgrind error that does not show up on CRAN.
  <testthat_skip_error>
  Memcheck:Cond
  fun:gregexpr_Regexc
@@ -32,3 +32,21 @@
  fun:getvar
  fun:bcEval
 }
+{
+ # This also doesn't seem to cause issues on CRAN, so suppress it.
+ <Memcheck_possible_leak>
+ Memcheck:Leak
+ match-leak-kinds: possible
+ fun:malloc
+ fun:libdeflate_alloc_compressor
+ fun:do_memCompress
+ fun:bcEval_loop
+ fun:bcEval
+ fun:Rf_eval
+ fun:R_execClosure
+ fun:applyClosure_core
+ fun:Rf_applyClosure
+ fun:Rf_eval
+ fun:do_set
+ fun:Rf_eval
+}
diff --git a/ci/scripts/r_sanitize.sh b/ci/scripts/r_sanitize.sh
@@ -49,7 +49,7 @@ export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp"
 
 # run tests
 pushd tests
-${R_BIN} < testthat.R > testthat.out 2>&1 || { cat testthat.out; exit 1; }
+${R_BIN} --no-save < testthat.R > testthat.out 2>&1 || { cat testthat.out; exit 1; }
 
 cat testthat.out
 if grep -q "runtime error" testthat.out; then
@@ -58,7 +58,7 @@ fi
 
 # run examples
 popd
-${R_BIN} -e 'library(arrow); testthat::test_examples(".")' >> examples.out 2>&1 || { cat examples.out; exit 1; }
+${R_BIN} --no-save -e 'library(arrow); testthat::test_examples(".")' >> examples.out 2>&1 || { cat examples.out; exit 1; }
 
 cat examples.out
 if grep -q "runtime error" examples.out; then

diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh
@@ -46,7 +46,12 @@ if [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then
  export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
  export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
 fi
-export _R_CHECK_COMPILATION_FLAGS_KNOWN_=${ARROW_R_CXXFLAGS}
+
+export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} ${ARROW_R_CXXFLAGS}"
+# These should generally be picked up, but are slightly wrong in rhub's containers it appears
+# https://github.com/r-hub/containers/pull/63
+export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} -Wno-parentheses -Werror=format-security -Wp,-D_FORTIFY_SOURCE=3"
+
 if [ "$ARROW_R_DEV" = "TRUE" ]; then
  # These are sometimes used in the Arrow C++ build and are not a problem
  export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} -Wno-attributes -msse4.2 -Wno-noexcept-type -Wno-subobject-linkage"

diff --git a/ci/scripts/r_valgrind.sh b/ci/scripts/r_valgrind.sh
@@ -33,7 +33,7 @@ ${R_BIN} CMD INSTALL ${INSTALL_ARGS} arrow*.tar.gz
 pushd tests
 
 # to generate suppression files run:
-# ${R_BIN} --vanilla -d "valgrind --tool=memcheck --leak-check=full --track-origins=yes --gen-suppressions=all --log-file=memcheck.log" -f testthat.supp
+# ${R_BIN} --vanilla -d "valgrind --tool=memcheck --leak-check=full --track-origins=yes --gen-suppressions=all --log-file=memcheck.log" -f testthat.R
 ${R_BIN} --vanilla -d "valgrind --tool=memcheck --leak-check=full --track-origins=yes --suppressions=/${1}/ci/etc/valgrind-cran.supp" -f testthat.R |& tee testthat.out
 
 # valgrind --error-exitcode=1 should return an erroring exit code that we can catch,

diff --git a/dev/tasks/r/github.linux.arrow.version.back.compat.yml b/dev/tasks/r/github.linux.arrow.version.back.compat.yml
@@ -73,6 +73,8 @@ jobs:
  config:
  # We use the R version that was released at the time of the arrow release in order
  # to make sure we can download binaries from RSPM.
+ - { old_arrow_version: '14.0.2.1', r: '4.3' }
+ - { old_arrow_version: '13.0.0.1', r: '4.3' }
  - { old_arrow_version: '12.0.1.1', r: '4.3' }
  - { old_arrow_version: '11.0.0.3', r: '4.2' }
  - { old_arrow_version: '10.0.1', r: '4.2' }

diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml
@@ -84,7 +84,7 @@ jobs:
  {{ macros.github_set_sccache_envvars()|indent(8)}}
  run: |
  cd arrow/r
- R CMD INSTALL --install-tests --no-test-load --no-docs --no-help --no-byte-compile arrow_with_deps.tar.gz
+ R CMD INSTALL --install-tests --no-test-load --no-byte-compile arrow_with_deps.tar.gz
  - name: Run the tests
  run: R -e 'if(tools::testInstalledPackage("arrow") != 0L) stop("There was a test failure.")'
  - name: Dump test logs

diff --git a/dev/tasks/r/github.linux.versions.yml b/dev/tasks/r/github.linux.versions.yml
@@ -30,9 +30,9 @@ jobs:
  r_version:
  # We test devel, release, and oldrel in regular CI.
  # This is for older versions
- - "3.6"
  - "4.0"
  - "4.1"
+ - "4.2"
  env:
  R_ORG: "rstudio"
  R_IMAGE: "r-base"

diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml
@@ -227,7 +227,7 @@ jobs:
  working-directory: 'arrow'
  extra-packages: cpp11
  - name: Set CRAN like openssl
- if: contains(matrix.platform.runs_on, 'arm64')
+ if: contains(matrix.platform.name, 'arm64')
  run: |
  # The arm64 runners contain openssl 1.1.1t in this path that is always included first so we need to override the
  # default setting of the brew --prefix as root dir to avoid version conflicts.
@@ -300,16 +300,14 @@ jobs:
  # an OS that is not in the allowlist, so we have to opt-in to use the
  # binary. Other env vars used in r_docker_configure.sh can be added
  # here (like devtoolset) and wired up in the later steps.
- - {image: "rhub/debian-clang-devel", libarrow_binary: "TRUE"}
+ - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE"}
  # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error)
  # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE"}
- - {image: "rhub/ubuntu-gcc-release"} # currently ubuntu-20.04 (focal)
- - {image: "rocker/r-ubuntu:22.04"} # openssl3
- - {image: "rocker/r-ver"} # whatever is latest ubuntu LTS
+ - {image: "rhub/ubuntu-release"} # currently ubuntu-22.04
  - {image: "rocker/r-ver:4.0.0"} # ubuntu-20.04
- - {image: "rocker/r-ver:3.6.3", libarrow_binary: "TRUE"} # debian:buster (10)
  - {image: "rstudio/r-base:4.1-focal"} # ubuntu-20.04
  - {image: "rstudio/r-base:4.2-centos7", devtoolset: "8"}
+ - {image: "rstudio/r-base:4.3-noble"}
  steps:
  # Get the arrow checkout just for the docker config scripts
  # Don't need submodules for this (hence false arg to macro): they fail on

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -888,12 +888,12 @@ tasks:
  - r-lib__libarrow__bin__darwin-arm64-openssl-3.0__arrow-{no_rc_r_version}\.zip
  - r-lib__libarrow__bin__darwin-x86_64-openssl-1.1__arrow-{no_rc_r_version}\.zip
  - r-lib__libarrow__bin__darwin-x86_64-openssl-3.0__arrow-{no_rc_r_version}\.zip
+ - r-pkg__bin__windows__contrib__4.4__arrow_{no_rc_r_version}\.zip
  - r-pkg__bin__windows__contrib__4.3__arrow_{no_rc_r_version}\.zip
- - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_r_version}\.zip
+ - r-pkg__bin__macosx__big-sur-x86_64__contrib__4.4__arrow_{no_rc_r_version}\.tgz
  - r-pkg__bin__macosx__big-sur-x86_64__contrib__4.3__arrow_{no_rc_r_version}\.tgz
- - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_r_version}\.tgz
+ - r-pkg__bin__macosx__big-sur-arm64__contrib__4.4__arrow_{no_rc_r_version}\.tgz
  - r-pkg__bin__macosx__big-sur-arm64__contrib__4.3__arrow_{no_rc_r_version}\.tgz
- - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_r_version}\.tgz
  - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz
 
 
@@ -1356,7 +1356,7 @@ tasks:
  r_tag: latest
  r_custom_ccache: true
 
-{% for r_org, r_image, r_tag in [("rhub", "ubuntu-gcc-release", "latest"),
+{% for r_org, r_image, r_tag in [("rhub", "ubuntu-release", "latest"),
  ("rocker", "r-ver", "latest"),
  ("rstudio", "r-base", "4.2-focal"),
  ("rstudio", "r-base", "4.1-opensuse153")] %}
@@ -1377,9 +1377,9 @@ tasks:
  template: r/azure.linux.yml
  params:
  r_org: rhub
- r_image: debian-gcc-devel-lto
+ r_image: gcc13
  r_tag: latest
- flags: '-e NOT_CRAN=false -e INSTALL_ARGS=--use-LTO'
+ flags: '-e INSTALL_ARGS=--use-LTO'
 
  # This one has -flto=auto
  test-r-ubuntu-22.04:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1459,8 +1459,8 @@ services:
  # (including building the C++ library) on any Docker image that contains R
  #
  # Usage:
- # R_ORG=rhub R_IMAGE=ubuntu-gcc-release R_TAG=latest docker-compose build r
- # R_ORG=rhub R_IMAGE=ubuntu-gcc-release R_TAG=latest docker-compose run r
+ # R_ORG=rhub R_IMAGE=ubuntu-release R_TAG=latest docker-compose build r
+ # R_ORG=rhub R_IMAGE=ubuntu-release R_TAG=latest docker-compose run r
  image: ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG}
  build:
  context: .
@@ -1523,6 +1523,7 @@ services:
  cache_from:
  - ${REPO}:r-rhub-fedora-clang-devel-latest
  args:
+ # TODO: change this to rhub/clang-asan
  base: rhub/fedora-clang-devel-san
  r_dev: ${ARROW_R_DEV}
  devtoolset_version: ${DEVTOOLSET_VERSION}

diff --git a/r/DESCRIPTION b/r/DESCRIPTION
@@ -21,7 +21,7 @@ Description: 'Apache' 'Arrow' <https://arrow.apache.org/> is a cross-language
  language-independent columnar memory format for flat and hierarchical data,
  organized for efficient analytic operations on modern hardware. This
  package provides an interface to the 'Arrow C++' library.
-Depends: R (>= 3.4)
+Depends: R (>= 4.0)
 License: Apache License (>= 2.0)
 URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/
 BugReports: https://github.com/apache/arrow/issues

diff --git a/r/R/dplyr-funcs-type.R b/r/R/dplyr-funcs-type.R
@@ -140,7 +140,7 @@ register_bindings_type_cast <- function() {
  fix.empty.names = TRUE,
  stringsAsFactors = FALSE) {
  # we need a specific value of stringsAsFactors because the default was
- # TRUE in R <= 3.6
+ # TRUE in R <= 3.6 and folks might still be cargoculting to stay in the past.
  if (!identical(stringsAsFactors, FALSE)) {
  arrow_not_supported("stringsAsFactors = TRUE")
  }

diff --git a/r/R/util.R b/r/R/util.R
@@ -15,20 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# for compatibility with R versions earlier than 4.0.0
-if (!exists("deparse1")) {
- deparse1 <- function(expr, collapse = " ", width.cutoff = 500L, ...) {
- paste(deparse(expr, width.cutoff, ...), collapse = collapse)
- }
-}
-
-# for compatibility with R versions earlier than 3.6.0
-if (!exists("str2lang")) {
- str2lang <- function(s) {
- parse(text = s, keep.source = FALSE)[[1]]
- }
-}
-
 oxford_paste <- function(x,
  conjunction = "and",
  quote = TRUE,

diff --git a/r/tests/testthat/test-Array.R b/r/tests/testthat/test-Array.R
@@ -818,11 +818,6 @@ test_that("Handling string data with embedded nuls", {
  )
  array_with_nul <- arrow_array(raws)$cast(utf8())
 
- # The behavior of the warnings/errors is slightly different with and without
- # altrep. Without it (i.e. 3.5.0 and below, the error would trigger immediately
- # on `as.vector()` where as with it, the error only happens on materialization)
- skip_on_r_older_than("3.6")
-
  # no error on conversion, because altrep laziness
  v <- expect_error(as.vector(array_with_nul), NA)
 

diff --git a/r/tests/testthat/test-RecordBatch.R b/r/tests/testthat/test-RecordBatch.R
@@ -595,14 +595,10 @@ test_that("RecordBatch supports cbind", {
  )
 
  # Rejects Table and ChunkedArray arguments
- if (getRversion() >= "4.0.0") {
- # R 3.6 cbind dispatch rules cause cbind to fall back to default impl if
- # there are multiple arguments with distinct cbind implementations
- expect_error(
- cbind(record_batch(a = 1:2), arrow_table(b = 3:4)),
- regexp = "Cannot cbind a RecordBatch with Tables or ChunkedArrays"
- )
- }
+ expect_error(
+ cbind(record_batch(a = 1:2), arrow_table(b = 3:4)),
+ regexp = "Cannot cbind a RecordBatch with Tables or ChunkedArrays"
+ )
  expect_error(
  cbind(record_batch(a = 1:2), b = chunked_array(1, 2)),
  regexp = "Cannot cbind a RecordBatch with Tables or ChunkedArrays"
@@ -622,10 +618,6 @@ test_that("Handling string data with embedded nuls", {
  batch_with_nul <- record_batch(a = 1:5, b = raws)
  batch_with_nul$b <- batch_with_nul$b$cast(utf8())
 
- # The behavior of the warnings/errors is slightly different with and without
- # altrep. Without it (i.e. 3.5.0 and below, the error would trigger immediately
- # on `as.vector()` where as with it, the error only happens on materialization)
- skip_on_r_older_than("3.6")
  df <- as.data.frame(batch_with_nul)
 
  expect_error(

diff --git a/r/tests/testthat/test-Table.R b/r/tests/testthat/test-Table.R
@@ -581,10 +581,6 @@ test_that("Table supports cbind", {
 })
 
 test_that("cbind.Table handles record batches and tables", {
- # R 3.6 cbind dispatch rules cause cbind to fall back to default impl if
- # there are multiple arguments with distinct cbind implementations
- skip_if(getRversion() < "4.0.0", "R 3.6 cbind dispatch rules prevent this behavior")
-
  expect_equal(
  cbind(arrow_table(a = 1L:2L), record_batch(b = 4:5)),
  arrow_table(a = 1L:2L, b = 4:5)

diff --git a/r/tests/testthat/test-altrep.R b/r/tests/testthat/test-altrep.R
@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-skip_on_r_older_than("3.6")
-
 test_that("altrep test functions do not include base altrep", {
  expect_false(is_arrow_altrep(1:10))
  expect_identical(test_arrow_altrep_is_materialized(1:10), NA)
@@ -373,6 +371,11 @@ test_that("altrep min/max/sum identical to R versions for double", {
  expect_altrep_roundtrip(x, max)
  expect_altrep_roundtrip(x, sum)
 
+ # On valgrind the NA_real_ is sometimes transformed to NaN
+ # https://stat.ethz.ch/pipermail/r-devel/2021-April/080683.html
+ # so we skip these there to avoid complicated NA == NaN logic,
+ # and they are tested on a number of other platforms / conditions
+ skip_on_linux_devel()
  x <- c(1, 2, NA_real_)
  expect_altrep_roundtrip(x, min, na.rm = TRUE)
  expect_altrep_roundtrip(x, max, na.rm = TRUE)

diff --git a/r/tests/testthat/test-chunked-array.R b/r/tests/testthat/test-chunked-array.R
@@ -475,11 +475,6 @@ test_that("Handling string data with embedded nuls", {
  )
  chunked_array_with_nul <- ChunkedArray$create(raws)$cast(utf8())
 
- # The behavior of the warnings/errors is slightly different with and without
- # altrep. Without it (i.e. 3.5.0 and below, the error would trigger immediately
- # on `as.vector()` where as with it, the error only happens on materialization)
- skip_on_r_older_than("3.6")
-
  v <- expect_error(as.vector(chunked_array_with_nul), NA)
 
  expect_error(

diff --git a/r/tests/testthat/test-dplyr-collapse.R b/r/tests/testthat/test-dplyr-collapse.R
@@ -185,16 +185,6 @@ See $.data for the source Arrow object",
  fixed = TRUE
  )
 
- skip_if(getRversion() < "3.6.0", "TODO investigate why these aren't equal")
- # On older R versions:
- # ── Failure (test-dplyr-collapse.R:172:3): Properties of collapsed query ────────
- # head(q, 1) %>% collect() not equal to tibble::tibble(lgl = FALSE, total = 8L, extra = 40).
- # Component "total": Mean relative difference: 0.3846154
- # Component "extra": Mean relative difference: 0.3846154
- # ── Failure (test-dplyr-collapse.R:176:3): Properties of collapsed query ────────
- # tail(q, 1) %>% collect() not equal to tibble::tibble(lgl = NA, total = 25L, extra = 125).
- # Component "total": Mean relative difference: 0.9230769
- # Component "extra": Mean relative difference: 0.9230769
  expect_equal(
  q %>%
  arrange(lgl) %>%