Merge branch 'main' into extend-read-parquet

.github/renovate.json

@@ -67,7 +67,7 @@
       "addLabels": ["druid"]
     },
     {
-      "matchPackagePatterns": ["pymysql", "mariadb"],
+      "matchPackagePatterns": ["mysqlclient", "mariadb"],
       "addLabels": ["mysql"]
     },
     {
@@ -78,6 +78,10 @@
       "matchPackagePatterns": ["pyspark"],
       "addLabels": ["pyspark"]
     },
+    {
+      "matchPackagePatterns": ["risingwave"],
+      "addLabels": ["risingwave"]
+    },
     {
       "matchPackagePatterns": ["snowflake-connector-python"],
       "addLabels": ["snowflake"]

.github/workflows/ibis-backends.yml

@@ -123,6 +123,7 @@ jobs:
               - sqlite
           - name: datafusion
             title: DataFusion
+            serial: true
             extras:
               - datafusion
           - name: polars
@@ -140,6 +141,7 @@ jobs:
               - polars
             sys-deps:
               - libgeos-dev
+              - default-libmysqlclient-dev
           - name: postgres
             title: PostgreSQL
             extras:
@@ -270,6 +272,7 @@ jobs:
                 - mysql
               sys-deps:
                 - libgeos-dev
+                - default-libmysqlclient-dev
           - os: windows-latest
             backend:
               name: clickhouse
@@ -650,24 +653,27 @@ jobs:
         run: docker compose logs
 
   test_pyspark:
-    name: PySpark ${{ matrix.pyspark-version }} ubuntu-latest python-${{ matrix.python-version }}
+    name: PySpark ${{ matrix.pyspark-minor-version }} ubuntu-latest python-${{ matrix.python-version }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         include:
           - python-version: "3.10"
             pyspark-version: "3.3.3"
+            pyspark-minor-version: "3.3"
             deps:
               - "'pandas@<2'"
               - "'numpy@<1.24'"
           - python-version: "3.11"
-            pyspark-version: "3.5"
+            pyspark-version: "3.5.2"
+            pyspark-minor-version: "3.5"
             deps:
               - "'pandas@>2'"
               - "'numpy@>1.24'"
           - python-version: "3.12"
-            pyspark-version: "3.5"
+            pyspark-version: "3.5.2"
+            pyspark-minor-version: "3.5"
             deps:
               - "'pandas@>2'"
               - "'numpy@>1.24'"
@@ -722,8 +728,7 @@ jobs:
 
       - name: install iceberg
         shell: bash
-        if: matrix.pyspark-version == '3.5'
-        run: pushd "$(poetry run python -c "import pyspark; print(pyspark.__file__.rsplit('/', 1)[0])")/jars" && curl -LO https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.5.2/iceberg-spark-runtime-3.5_2.12-1.5.2.jar
+        run: just download-iceberg-jar ${{ matrix.pyspark-minor-version }}
 
       - name: run tests
         run: just ci-check -m pyspark

compose.yaml

@@ -99,7 +99,7 @@ services:
       - trino
 
   minio:
-    image: bitnami/minio:2024.9.9
+    image: bitnami/minio:2024.9.13
     environment:
       MINIO_ROOT_USER: accesskey
       MINIO_ROOT_PASSWORD: secretkey
@@ -161,7 +161,7 @@ services:
       test:
         - CMD-SHELL
         - trino --output-format null --execute 'show schemas in hive; show schemas in memory'
-    image: trinodb/trino:457
+    image: trinodb/trino:458
     ports:
       - 8080:8080
     networks:
@@ -204,7 +204,7 @@ services:
       - druid
 
   druid-coordinator:
-    image: apache/druid:30.0.0
+    image: apache/druid:30.0.1
     hostname: coordinator
     container_name: coordinator
     volumes:
@@ -229,7 +229,7 @@ services:
       - druid
 
   druid-broker:
-    image: apache/druid:30.0.0
+    image: apache/druid:30.0.1
     hostname: broker
     container_name: broker
     volumes:
@@ -257,7 +257,7 @@ services:
       - druid
 
   druid-historical:
-    image: apache/druid:30.0.0
+    image: apache/druid:30.0.1
     hostname: historical
     container_name: historical
     volumes:
@@ -284,7 +284,7 @@ services:
       - druid
 
   druid-middlemanager:
-    image: apache/druid:30.0.0
+    image: apache/druid:30.0.1
     hostname: middlemanager
     container_name: middlemanager
     volumes:
@@ -312,7 +312,7 @@ services:
       - druid
 
   druid:
-    image: apache/druid:30.0.0
+    image: apache/druid:30.0.1
     hostname: router
     container_name: router
     volumes:
@@ -557,7 +557,7 @@ services:
       - impala
 
   risingwave:
-    image: ghcr.io/risingwavelabs/risingwave:v1.10.1
+    image: ghcr.io/risingwavelabs/risingwave:v2.0.0
     command: "standalone --meta-opts=\" \
       --advertise-addr 0.0.0.0:5690 \
       --backend mem \

conda/environment-arm64-flink.yml

@@ -33,7 +33,7 @@ dependencies:
   - pyarrow-hotfix >=0.4
   - pydata-google-auth
   - pydruid >=0.6.5
-  - pymysql >=1
+  - mysqlclient >=2.2.4
   - pyspark >=3
   - python-dateutil >=2.8.2
   - python-duckdb >=0.8.1

conda/environment-arm64.yml

@@ -33,7 +33,7 @@ dependencies:
   - pyarrow-hotfix >=0.4
   - pydata-google-auth
   - pydruid >=0.6.5
-  - pymysql >=1
+  - mysqlclient >=2.2.4
   - pyodbc >=4.0.39
   - pyspark >=3
   - python-dateutil >=2.8.2

conda/environment.yml

@@ -33,7 +33,7 @@ dependencies:
   - pyarrow-hotfix >=0.4
   - pydata-google-auth
   - pydruid >=0.6.5
-  - pymysql >=1
+  - mysqlclient >=2.2.4
   - pyodbc >=4.0.39
   - pyspark >=3
   - python >=3.10

docker/trino/catalog/hive.properties

@@ -1,12 +1,18 @@
 connector.name=hive
 
+# this enables the native s3 connector and the various s3.* properties
+fs.native-s3.enabled=true
+
+# region is bogus, but trino requires it
+s3.region=us-east-1
+s3.endpoint=http://minio:9000
+s3.aws-access-key=accesskey
+s3.aws-secret-key=secretkey
+s3.path-style-access=true
+
 hive.ignore-absent-partitions=true
 hive.metastore.thrift.delete-files-on-drop=true
 hive.metastore.uri=thrift://hive-metastore:9083
 hive.metastore.username=admin
 hive.non-managed-table-writes-enabled=true
-hive.s3.aws-access-key=accesskey
-hive.s3.aws-secret-key=secretkey
-hive.s3.endpoint=http://minio:9000
-hive.s3.path-style-access=true
 hive.timestamp-precision=MICROSECONDS

docs/_quarto.yml

@@ -220,6 +220,7 @@ quartodoc:
           contents:
             - name: Table
               package: ibis.expr.types.relations
+              include_inherited: true
             - name: GroupedTable
               package: ibis.expr.types.groupby
             - name: read_csv