Merge branch 'main' into parquet/support-write-bloom-filter

# Conflicts: # cpp/src/parquet/page_index.h
mapleFU · Oct 2, 2023 · 7fd47be · 7fd47be
2 parents 8e9cb16 + b0eb021
commit 7fd47be
Show file tree

Hide file tree

Showing 444 changed files with 12,263 additions and 2,699 deletions.
diff --git a/.env b/.env
@@ -58,8 +58,8 @@ CUDA=11.0.3
 DASK=latest
 DOTNET=7.0
 GCC_VERSION=""
-GO=1.17
-STATICCHECK=v0.2.2
+GO=1.19.13
+STATICCHECK=v0.4.5
 HDFS=3.2.1
 JDK=8
 KARTOTHEK=latest

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -54,28 +54,23 @@ jobs:
         include:
           - arch-label: AMD64
             arch: amd64
-            go: 1.17
+            go: 1.19
             runs-on: ubuntu-latest
-            staticcheck: v0.2.2
           - arch-label: AMD64
             arch: amd64
-            go: 1.18
+            go: '1.20'
             runs-on: ubuntu-latest
-            staticcheck: v0.3.3
           - arch-label: ARM64
             arch: arm64v8
-            go: 1.17
-            staticcheck: v0.2.2
+            go: 1.19
             runs-on: ["self-hosted", "arm", "linux"]
           - arch-label: ARM64
             arch: arm64v8
-            go: 1.18
-            staticcheck: v0.3.3
+            go: '1.20'
             runs-on: ["self-hosted", "arm", "linux"]
     env:
       ARCH: ${{ matrix.arch }}
       GO: ${{ matrix.go }}
-      STATICCHECK: ${{ matrix.staticcheck }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -145,7 +140,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: 1.19
           cache: true
           cache-dependency-path: go/go.sum
       - name: Run build
@@ -161,15 +156,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     env:
       GO: ${{ matrix.go }}
-      STATICCHECK: ${{ matrix.staticcheck }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -208,15 +197,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     env:
       GO: ${{ matrix.go }}
-      STATICCHECK: ${{ matrix.staticcheck }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -249,16 +232,11 @@ jobs:
     name: AMD64 Windows 2019 Go ${{ matrix.go }}
     runs-on: windows-2019
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 15
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -272,7 +250,10 @@ jobs:
           cache: true
           cache-dependency-path: go/go.sum
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }}
+        shell: bash
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)
@@ -288,12 +269,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -306,8 +282,10 @@ jobs:
           go-version: ${{ matrix.go }}
           cache: true
           cache-dependency-path: go/go.sum
-      - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }}
+      - name: Install staticcheck      
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)
@@ -349,12 +327,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     env:
       ARROW_GO_TESTCGO: "1"
     steps:
@@ -373,7 +346,9 @@ jobs:
         shell: bash
         run: brew install apache-arrow pkg-config
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }}
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Add To pkg config path
         shell: bash
         run: |
@@ -430,11 +405,14 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.18'
+          go-version: '1.19'
           cache: true
           cache-dependency-path: go/go.sum
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@v0.3.3
+        shell: bash
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)

diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
@@ -53,6 +53,8 @@ jobs:
         run: sudo apt-get install ninja-build
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v1
+        with:
+          release: R2023a
       - name: Install ccache
         run: sudo apt-get install ccache
       - name: Setup ccache
@@ -99,6 +101,8 @@ jobs:
         run: brew install ninja
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v1
+        with:
+          release: R2023a
       - name: Install ccache
         run: brew install ccache
       - name: Setup ccache
@@ -135,6 +139,8 @@ jobs:
           fetch-depth: 0
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v1
+        with:
+          release: R2023a
       - name: Download Timezone Database
         shell: bash
         run: ci/scripts/download_tz_database.sh

diff --git a/c_glib/arrow-glib/compute.cpp b/c_glib/arrow-glib/compute.cpp
@@ -3346,7 +3346,7 @@ garrow_set_lookup_options_get_property(GObject *object,
     g_value_set_object(value, priv->value_set);
     break;
   case PROP_SET_LOOKUP_OPTIONS_SKIP_NULLS:
-    g_value_set_boolean(value, options->skip_nulls);
+    g_value_set_boolean(value, options->skip_nulls.has_value() && options->skip_nulls.value());
     break;
   default:
     G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
@@ -3398,13 +3398,11 @@ garrow_set_lookup_options_class_init(GArrowSetLookupOptionsClass *klass)
    *
    * Since: 6.0.0
    */
-  spec = g_param_spec_boolean("skip-nulls",
-                              "Skip NULLs",
-                              "Whether NULLs are skipped or not",
-                              options.skip_nulls,
-                              static_cast<GParamFlags>(G_PARAM_READWRITE));
-  g_object_class_install_property(gobject_class,
-                                  PROP_SET_LOOKUP_OPTIONS_SKIP_NULLS,
+  auto skip_nulls = (options.skip_nulls.has_value() && options.skip_nulls.value());
+  spec =
+      g_param_spec_boolean("skip-nulls", "Skip NULLs", "Whether NULLs are skipped or not",
+                           skip_nulls, static_cast<GParamFlags>(G_PARAM_READWRITE));
+  g_object_class_install_property(gobject_class, PROP_SET_LOOKUP_OPTIONS_SKIP_NULLS,
                                   spec);
 }
 
@@ -6458,9 +6456,10 @@ garrow_set_lookup_options_new_raw(
       arrow_copied_options.get());
   auto value_set =
     garrow_datum_new_raw(&(arrow_copied_set_lookup_options->value_set));
+  auto skip_nulls = (arrow_options->skip_nulls.has_value() && arrow_options->skip_nulls.value());
   auto options = g_object_new(GARROW_TYPE_SET_LOOKUP_OPTIONS,
                               "value-set", value_set,
-                              "skip-nulls", arrow_options->skip_nulls,
+                              "skip-nulls", skip_nulls,
                               NULL);
   return GARROW_SET_LOOKUP_OPTIONS(options);
 }

diff --git a/c_glib/test/test-array-datum.rb b/c_glib/test/test-array-datum.rb
@@ -61,7 +61,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("Array", @datum.to_s)
+    assert_equal("Array([\n" + "  true,\n" + "  false\n" + "])", @datum.to_s)
   end
 
   def test_value

diff --git a/c_glib/test/test-chunked-array-datum.rb b/c_glib/test/test-chunked-array-datum.rb
@@ -49,7 +49,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("ChunkedArray", @datum.to_s)
+    assert_equal("ChunkedArray([\n" + "  [\n" + "    true,\n" + "    false\n" + "  ]\n" + "])", @datum.to_s)
   end
 
   def test_value

diff --git a/c_glib/test/test-record-batch-datum.rb b/c_glib/test/test-record-batch-datum.rb
@@ -49,7 +49,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("RecordBatch", @datum.to_s)
+    assert_equal("RecordBatch(visible:   [\n" + "    true,\n" + "    false\n" + "  ]\n" + ")", @datum.to_s)
   end
 
   def test_value

diff --git a/c_glib/test/test-scalar-datum.rb b/c_glib/test/test-scalar-datum.rb
@@ -60,7 +60,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("Scalar", @datum.to_s)
+    assert_equal("Scalar(true)", @datum.to_s)
   end
 
   def test_value

diff --git a/c_glib/test/test-table-datum.rb b/c_glib/test/test-table-datum.rb
@@ -49,7 +49,16 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("Table", @datum.to_s)
+    assert_equal("Table(visible: bool\n" +
+    "----\n" +
+    "visible:\n" +
+    "  [\n" +
+    "    [\n" +
+    "      true,\n" +
+    "      false\n" +
+    "    ]\n" +
+    "  ]\n" +
+    ")", @datum.to_s)
   end
 
   def test_value

diff --git a/ci/conda_env_archery.txt b/ci/conda_env_archery.txt
@@ -25,7 +25,7 @@ jira
 pygit2
 pygithub
 ruamel.yaml
-setuptools_scm
+setuptools_scm<8.0.0
 toolz
 
 # benchmark

diff --git a/ci/conda_env_crossbow.txt b/ci/conda_env_crossbow.txt
@@ -21,5 +21,5 @@ jinja2
 jira
 pygit2
 ruamel.yaml
-setuptools_scm
+setuptools_scm<8.0.0
 toolz
diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt
@@ -28,4 +28,4 @@ pytest-faulthandler
 pytest-lazy-fixture
 s3fs>=2021.8.0
 setuptools
-setuptools_scm
+setuptools_scm<8.0.0
diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt
@@ -20,7 +20,7 @@ breathe
 doxygen
 ipython
 numpydoc
-pydata-sphinx-theme==0.8
+pydata-sphinx-theme
 sphinx-autobuild
 sphinx-design
 sphinx-copybutton

diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile
@@ -24,7 +24,7 @@ ARG maven=3.5
 ARG node=16
 ARG yarn=1.22
 ARG jdk=8
-ARG go=1.15
+ARG go=1.19.13
 
 # Install Archery and integration dependencies
 COPY ci/conda_env_archery.txt /arrow/ci/

diff --git a/ci/docker/conda-python-cython2.dockerfile b/ci/docker/conda-python-cython2.dockerfile
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG repo
+ARG arch
+ARG python=3.8
+FROM ${repo}:${arch}-conda-python-${python}
+
+RUN mamba install -q -y "cython<3" && \
+    mamba clean --all
diff --git a/ci/docker/debian-11-go.dockerfile b/ci/docker/debian-11-go.dockerfile
@@ -16,8 +16,8 @@
 # under the License.
 
 ARG arch=amd64
-ARG go=1.17
-ARG staticcheck=v0.2.2
+ARG go=1.19
+ARG staticcheck=v0.4.5
 FROM ${arch}/golang:${go}-bullseye
 
 # FROM collects all the args, get back the staticcheck version arg