Merge branch 'apache:main' into master

fatemehp · Oct 3, 2023 · 64f9d3b · 64f9d3b
2 parents 2d073bf + 5514b22
commit 64f9d3b
Show file tree

Hide file tree

Showing 1,729 changed files with 58,540 additions and 16,575 deletions.
diff --git a/.env b/.env
@@ -58,8 +58,8 @@ CUDA=11.0.3
 DASK=latest
 DOTNET=7.0
 GCC_VERSION=""
-GO=1.17
-STATICCHECK=v0.2.2
+GO=1.19.13
+STATICCHECK=v0.4.5
 HDFS=3.2.1
 JDK=8
 KARTOTHEK=latest
@@ -98,7 +98,7 @@ VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"    # 2023.04.15 Release
 # ci/docker/python-wheel-windows-vs2017.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02
 
 # Use conanio/${CONAN} for "docker-compose run --rm conan". See
 # https://github.com/conan-io/conan-docker-tools#readme for available

diff --git a/.gitattributes b/.gitattributes
@@ -1,9 +1,11 @@
+cpp/src/arrow/util/bpacking_*_generated.h linguist-generated=true
+cpp/src/generated/*.cpp linguist-generated=true
+cpp/src/generated/*.h linguist-generated=true
+go/**/*.s linguist-generated=true
+go/arrow/unionmode_string.go linguist-generated=true
 r/R/RcppExports.R linguist-generated=true
 r/R/arrowExports.R linguist-generated=true
 r/src/RcppExports.cpp linguist-generated=true
 r/src/arrowExports.cpp linguist-generated=true
 r/man/*.Rd linguist-generated=true
-cpp/src/generated/*.h linguist-generated=true
 r/NEWS.md merge=union
-go/**/*.s linguist-generated=true
-go/arrow/unionmode_string.go linguist-generated=true
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -23,6 +23,12 @@ updates:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [CI] "
+  - package-ecosystem: "npm"
+    directory: "/js/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "MINOR: [JS] "
   - package-ecosystem: "nuget"
     directory: "/csharp/"
     schedule:

diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
@@ -50,7 +50,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Git Fixup

diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml
@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: arrow
           # fetch the tags for version number generation
@@ -60,7 +60,7 @@ jobs:
     if: startsWith(github.event.comment.body, '@github-actions autotune')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: r-lib/actions/pr-fetch@v2
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -158,7 +158,7 @@ jobs:
     if: startsWith(github.event.comment.body, '@github-actions rebase')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: r-lib/actions/pr-fetch@v2
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -23,22 +23,26 @@ on:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
       - 'ci/scripts/util_*'
       - 'cpp/**'
+      - 'docker-compose.yml'
       - 'format/Flight.proto'
   pull_request:
     paths:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
       - 'ci/scripts/util_*'
       - 'cpp/**'
+      - 'docker-compose.yml'
       - 'format/Flight.proto'
 
 concurrency:
@@ -92,7 +96,7 @@ jobs:
       UBUNTU: ${{ matrix.ubuntu }}
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive
@@ -134,7 +138,7 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive
@@ -162,10 +166,10 @@ jobs:
       ARROW_HDFS: ON
       ARROW_HOME: /usr/local
       ARROW_JEMALLOC: ON
-      # TODO(kszucs): link error in the tests
-      ARROW_ORC: OFF
+      ARROW_ORC: ON
       ARROW_PARQUET: ON
       ARROW_S3: ON
+      ARROW_SUBSTRAIT: ON
       ARROW_WITH_BROTLI: ON
       ARROW_WITH_BZ2: ON
       ARROW_WITH_LZ4: ON
@@ -182,7 +186,7 @@ jobs:
           sysctl -a | grep cpu
           sysctl -a | grep "hw.optional"
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive
@@ -201,6 +205,8 @@ jobs:
             ci/scripts/install_minio.sh latest /usr/local
       - name: Install Google Cloud Storage Testbench
         run: ci/scripts/install_gcs_testbench.sh default
+      - name: Install Azurite Storage Emulator
+        run: ci/scripts/install_azurite.sh
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh
@@ -254,6 +260,7 @@ jobs:
       ARROW_ORC: ON
       ARROW_PARQUET: ON
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
+      ARROW_SUBSTRAIT: ON
       ARROW_USE_GLOG: OFF
       ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
       ARROW_WITH_BROTLI: OFF
@@ -269,7 +276,6 @@ jobs:
       CMAKE_INSTALL_LIBDIR: bin
       CMAKE_INSTALL_PREFIX: /usr
       CMAKE_UNITY_BUILD: ON
-      NPROC: 3
     steps:
       - name: Disable Crash Dialogs
         run: |
@@ -284,7 +290,7 @@ jobs:
       - name: Install Dependencies
         run: choco install -y --no-progress openssl
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive
@@ -356,6 +362,7 @@ jobs:
       ARROW_JEMALLOC: OFF
       ARROW_PARQUET: ON
       ARROW_S3: ON
+      ARROW_SUBSTRAIT: ON
       ARROW_USE_GLOG: OFF
       ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
       ARROW_WITH_BROTLI: ON
@@ -368,7 +375,7 @@ jobs:
       # Don't use preinstalled Boost by empty BOOST_ROOT and
       # -DBoost_NO_BOOST_CMAKE=ON
       BOOST_ROOT: ""
-      CMAKE_ARGS: >-
+      ARROW_CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/${{ matrix.msystem_lower}}
         -DBoost_NO_BOOST_CMAKE=ON
         -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON
@@ -386,7 +393,7 @@ jobs:
             /d 1 `
             /f
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive

diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml
@@ -53,7 +53,7 @@ jobs:
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Source Link
@@ -81,7 +81,7 @@ jobs:
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Source Link
@@ -108,7 +108,7 @@ jobs:
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Source Link

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -37,7 +37,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Setup Python
@@ -70,21 +70,21 @@ jobs:
     name: Source Release and Merge Script on ${{ matrix.runs-on }}
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 10
+    timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
         runs-on:
           - macos-latest
           - ubuntu-latest
     env:
-      GIT_AUTHOR_NAME: Github Actions
-      GIT_AUTHOR_EMAIL: github@actions
-      GIT_COMMITTER_NAME: Github Actions
-      GIT_COMMITTER_EMAIL: github@actions
+      GIT_AUTHOR_NAME: "github-actions[bot]"
+      GIT_AUTHOR_EMAIL: "github-actions[bot]@users.noreply.github.com"
+      GIT_COMMITTER_NAME: "github-actions[bot]"
+      GIT_COMMITTER_EMAIL: "github-actions[bot]@users.noreply.github.com"
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Python

diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml
@@ -43,7 +43,7 @@ jobs:
     name: Process
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: apache/arrow
           ref: main

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -30,17 +30,20 @@ env:
 jobs:
 
   complete:
-    name: AMD64 Ubuntu 20.04 Complete Documentation
+    name: AMD64 Ubuntu 22.04 Complete Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 150
     env:
-      UBUNTU: "20.04"
+      UBUNTU: "22.04"
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Free up disk space
+        run: |
+          ci/scripts/util_free_space.sh
       - name: Cache Docker Volumes
         uses: actions/cache@v3
         with:

diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
@@ -27,6 +27,7 @@ on:
       - 'ci/docker/conda-python.dockerfile'
       - 'ci/scripts/cpp_build.sh'
       - 'ci/scripts/python_build.sh'
+      - 'docker-compose.yml'
 
 permissions:
   contents: read
@@ -41,12 +42,12 @@ jobs:
     name: AMD64 Conda Python 3.9 Sphinx Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 45
+    timeout-minutes: 60
     env:
       PYTHON: "3.9"
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Cache Docker Volumes