Merge branch 'branch-24.08' into raft-cpu-miniforge

.github/ISSUE_TEMPLATE/bug_report.md

@@ -21,7 +21,7 @@ A clear and concise description of what you expected to happen.
  - Method of cuDF install: [conda, Docker, or from source]
    - If method of install is [Docker], provide `docker pull` & `docker run` commands used
  - Please run and attach the output of the `cudf/print_env.sh` script to gather relevant environment details
- 
+
 
 **Additional context**
 Add any other context about the problem here.

.github/workflows/build-image.yml

@@ -24,9 +24,6 @@ on:
       RAPIDS_VER:
         required: true
         type: string
-      DASK_SQL_VER:
-        required: true
-        type: string
       BASE_TAG:
         required: true
         type: string
@@ -52,7 +49,6 @@ jobs:
         LINUX_VER: ["${{ inputs.LINUX_VER }}"]
         PYTHON_VER: ["${{ inputs.PYTHON_VER }}"]
         RAPIDS_VER: ["${{ inputs.RAPIDS_VER }}"]
-        DASK_SQL_VER: ["${{ inputs.DASK_SQL_VER }}"]
       fail-fast: false
     runs-on: "linux-${{ matrix.ARCH }}-cpu4"
     steps:
@@ -90,7 +86,7 @@ jobs:
           driver: docker
           endpoint: builders
       - name: Build base image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: context
           file: Dockerfile
@@ -104,10 +100,9 @@ jobs:
             LINUX_VER=${{ inputs.LINUX_VER }}
             PYTHON_VER=${{ inputs.PYTHON_VER }}
             RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-            DASK_SQL_VER=${{ inputs.DASK_SQL_VER }}
           tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
       - name: Build notebooks image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: context
           file: Dockerfile
@@ -121,10 +116,9 @@ jobs:
             LINUX_VER=${{ inputs.LINUX_VER }}
             PYTHON_VER=${{ inputs.PYTHON_VER }}
             RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-            DASK_SQL_VER=${{ inputs.DASK_SQL_VER }}
           tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
       - name: Build RAFT ANN Benchmarks GPU image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: context
           file: raft-ann-bench/gpu/Dockerfile
@@ -138,7 +132,7 @@ jobs:
             RAPIDS_VER=${{ inputs.RAPIDS_VER }}
           tags: ${{ inputs.RAFT_ANN_BENCH_TAG }}-${{ matrix.ARCH }}
       - name: Build RAFT ANN Benchmarks GPU with datasets image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: context
           file: raft-ann-bench/gpu/Dockerfile
@@ -152,8 +146,8 @@ jobs:
             RAPIDS_VER=${{ inputs.RAPIDS_VER }}
           tags: ${{ inputs.RAFT_ANN_BENCH_DATASETS_TAG }}-${{ matrix.ARCH }}
       - name: Build RAFT ANN Benchmarks CPU image
-        if: inputs.CUDA_VER == '12.2.2' # we don't need to build CPU packages for different CUDA versions.
-        uses: docker/build-push-action@v4
+        if: inputs.CUDA_VER == '12.5.1' # we don't need to build CPU packages for different CUDA versions.
+        uses: docker/build-push-action@v6
         with:
           context: context
           file: raft-ann-bench/cpu/Dockerfile

.github/workflows/build-test-publish-images.yml

@@ -133,7 +133,6 @@ jobs:
       LINUX_VER: ${{ matrix.LINUX_VER }}
       PYTHON_VER: ${{ matrix.PYTHON_VER }}
       RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-      DASK_SQL_VER: ${{ matrix.DASK_SQL_VER }}
       BASE_TAG:
         "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
         ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\

.github/workflows/pr.yml

@@ -10,7 +10,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  pr-builder:
+    needs:
+      - checks
+      - docker
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Run pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit run --all-files
   docker:
+    needs: [checks]
     uses: ./.github/workflows/build-test-publish-images.yml
     with:
       build_type: pull-request

.github/workflows/publish.yml

@@ -38,7 +38,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Update DockerHub README for ${{ matrix.repo_name }}
-        uses: peter-evans/dockerhub-description@v3
+        uses: peter-evans/dockerhub-description@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}

.github/workflows/release-to-ngc.yml

@@ -20,7 +20,7 @@ jobs:
         id: generate-matrix
         run: |
           #!/bin/bash
-          matrix=$(yq 'del(.DASK_SQL_VER)' matrix.yaml | yq -o json | jq -c)
+          matrix=$(yq '.' matrix.yaml | yq -o json | jq -c)
           echo "matrix=${matrix}" | tee -a ${GITHUB_OUTPUT}
 
   copy-images:

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.2
+    hooks:
+      - id: ruff
+        args: ["--config", "pyproject.toml"]

CONTRIBUTING.md

@@ -15,4 +15,3 @@ To build just the `base` image with default arguments: `docker buildx build --pu
 - `CUDA_VER` - Version of CUDA to use. Should be `major.minor.patch`
 - `PYTHON_VER` - Version of Python to use. Should be `major.minor`
 - `RAPIDS_VER` - Version of RAPIDS to use. Should be `YY.MM`
-- `DASK_SQL_VER` - Version of `dask-sql` to use. Should be `YYYY.M.P`

Dockerfile

@@ -1,21 +1,19 @@
 # syntax=docker/dockerfile:1
 
-ARG CUDA_VER=12.0.1
-ARG PYTHON_VER=3.11
+ARG CUDA_VER=unset
+ARG PYTHON_VER=unset
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_DISTRO_VER=22.04
 ARG LINUX_VER=${LINUX_DISTRO}${LINUX_DISTRO_VER}
 
 ARG RAPIDS_VER=24.08
-ARG DASK_SQL_VER=2024.5.0
 
 # Gather dependency information
 FROM rapidsai/ci-conda:latest AS dependencies
 ARG CUDA_VER
 ARG PYTHON_VER
 
 ARG RAPIDS_VER
-ARG DASK_SQL_VER
 
 ARG RAPIDS_BRANCH="branch-${RAPIDS_VER}"
 
@@ -36,12 +34,11 @@ EOF
 
 
 # Base image
-FROM rapidsai/miniforge-cuda:cuda${CUDA_VER}-base-${LINUX_VER}-py${PYTHON_VER} as base
+FROM rapidsai/miniforge-cuda:cuda${CUDA_VER}-base-${LINUX_VER}-py${PYTHON_VER} AS base
 ARG CUDA_VER
 ARG PYTHON_VER
 
 ARG RAPIDS_VER
-ARG DASK_SQL_VER
 
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
@@ -56,7 +53,6 @@ COPY condarc /opt/conda/.condarc
 RUN <<EOF
 mamba install -y -n base \
     "rapids=${RAPIDS_VER}.*" \
-    "dask-sql=${DASK_SQL_VER%.*}.*" \
     "python=${PYTHON_VER}.*" \
     "cuda-version=${CUDA_VER%.*}.*" \
     ipython
@@ -71,7 +67,7 @@ CMD ["ipython"]
 
 
 # Notebooks image
-FROM base as notebooks
+FROM base AS notebooks
 
 ARG CUDA_VER
 ARG LINUX_DISTRO
@@ -94,11 +90,10 @@ EOF
 
 RUN <<EOF
 mamba install -y -n base \
-        "jupyterlab=3" \
-        dask-labextension
-pip install "jupyterlab-nvdashboard==0.9.*"
+    "jupyterlab=4" \
+    dask-labextension \
+    jupyterlab-nvdashboard
 conda clean -afy
-pip cache purge
 EOF
 
 # Disable the JupyterLab announcements
@@ -138,7 +133,7 @@ LABEL com.nvidia.workbench.package-manager-environment.type="conda"
 LABEL com.nvidia.workbench.package-manager.apt.binary="/usr/bin/apt"
 LABEL com.nvidia.workbench.package-manager.apt.installed-packages=""
 LABEL com.nvidia.workbench.package-manager.conda3.binary="/opt/conda/bin/conda"
-LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cuspatial cuxfilter cucim xgboost dask-sql jupyterlab"
+LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cuspatial cuxfilter cucim xgboost jupyterlab"
 LABEL com.nvidia.workbench.package-manager.pip.binary="/opt/conda/bin/pip"
 LABEL com.nvidia.workbench.package-manager.pip.installed-packages="jupyterlab-nvdashboard"
 LABEL com.nvidia.workbench.programming-languages="python3"

README.md

@@ -11,7 +11,7 @@ There are two image types: `base` ([`rapidsai/base`](https://hub.docker.com/r/ra
 
 This image can be found here: https://hub.docker.com/r/rapidsai/base
 
-It contains the basic installation of RAPIDS and [`dask-sql`](https://github.com/dask-contrib/dask-sql). By default it starts an `ipython` REPL.
+It contains the basic installation of RAPIDS. By default it starts an `ipython` REPL.
 
 ### Notebooks image
 

ci/compute-matrix.jq

@@ -1,13 +1,5 @@
 def compute_arch($x):
-  ["amd64"] |
-  if
-    $x.LINUX_VER != "ubuntu20.04" # Dask-sql arm64 requires glibc >=2.32
-  then
-    . + ["arm64"]
-  else
-    .
-  end |
-  $x + {ARCHES: .};
+  $x + {ARCHES: ["amd64", "arm64"]};
 
 def compute_ubuntu_version($x):
   if

context/notebooks.sh

@@ -3,7 +3,6 @@
 # Clones repos with notebooks & compiles notebook test dependencies
 # Requires environment variables:
 #    RAPIDS_BRANCH
-#    DASK_SQL_VER
 #    CUDA_VER
 #    PYTHON_VER
 
@@ -41,6 +40,5 @@ done
 
 pushd "/dependencies"
 conda-merge ./*.yaml |
-    yq ".dependencies += [\"dask-sql==${DASK_SQL_VER%.*}.*\"]" | # Ensure dask-sql dependency is not altered
     yq '.channels = load("/condarc").channels' |                 # Use channels provided by CI, not repos
     tee /test_notebooks_dependencies.yaml

context/test_notebooks.py

@@ -5,9 +5,7 @@
 import sys
 import timeit
 from typing import Iterable
-import nbconvert
 import nbformat
-from datetime import datetime
 from nbconvert.preprocessors import ExecutePreprocessor
 import yaml
 
@@ -29,12 +27,15 @@
     # following nbs are marked as skipped
     'cugraph/algorithms/layout/Force-Atlas2.ipynb',
     'cuspatial/binary_predicates.ipynb',
-    'cuspatial/cuproj_benchmark.ipynb'
+    'cuspatial/cuproj_benchmark.ipynb',
+    # context on these being skipped: https://github.com/rapidsai/cuspatial/pull/1407
+    'cuspatial/cuspatial_api_examples.ipynb',
+    'cuspatial/nyc_taxi_years_correlation.ipynb'
 ]
 
 
 def get_notebooks(directory: str) -> Iterable[str]:
-    for root, dirs, files in os.walk(directory):
+    for root, _, files in os.walk(directory):
         for file in files:
             if (
                 file.endswith(".ipynb")
@@ -69,14 +70,15 @@ def test_notebook(notebook_file, executed_nb_file):
         warnings = []
         outputs = []
 
+
         # use nbconvert to run the notebook natively
         ep = ExecutePreprocessor(timeout=600, kernel_name="python3", allow_errors=True)
+        task_init = timeit.default_timer()
         try:
-            task_init = timeit.default_timer()
-            nb, nb_resources = ep.preprocess(nb, {"metadata": {"path": ""}})
-            execution_time = timeit.default_timer() - task_init
+            nb, _ = ep.preprocess(nb, {"metadata": {"path": ""}})
         except Exception as e:
             errors.append(e)
+        execution_time = timeit.default_timer() - task_init
 
         with open(executed_nb_file, "w", encoding="utf-8") as f:
             nbformat.write(nb, f)
@@ -152,7 +154,7 @@ def test_notebook(notebook_file, executed_nb_file):
         print(f"Input must be a directory. Got: {ns.input}")
         sys.exit(1)
 
-    notebooks = sorted(list(get_notebooks(ns.input)))
+    notebooks = sorted(get_notebooks(ns.input))
     print(f"{len(notebooks)} Notebooks to be tested:")
     for notebook in notebooks:
         print(notebook)

dockerhub-readme.md

@@ -22,7 +22,6 @@ RAPIDS Libraries included in the images:
 - `cuxfilter`
 - `cuCIM`
 - `xgboost`
-- `dask-sql`
 
 
 ### Image Types