Dependency update for May 2024 - black, boost-pp, cv-cuda, pybind11, rapidjson

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      - run: pip install  flake8 bandit "black[jupyter]"==24.2.0
+      - run: pip install  flake8 bandit "black[jupyter]"==24.4.2
       - run: black --check --verbose ${{ env.PYTHON_SECURITY_LINT_PATHS }} ${{ env.PYTHON_LINT_PATHS }} ${{ env.AUTOGRAPH_LINT_PATHS }}
       - run: flake8 --config=.flake8 ${{ env.PYTHON_SECURITY_LINT_PATHS }} ${{ env.PYTHON_LINT_PATHS }}
       - run: flake8 --config=.flake8.ag ${{ env.AUTOGRAPH_LINT_PATHS }}

DALI_DEPS_VERSION

@@ -1 +1 @@
-f0fe9cb92317a788b7dd9c01d73f79bb6aace349
+22cb369c551d2ba5fd1cfe2b667965d968258cda

conda/config/conda_build_config.yaml

@@ -2,3 +2,8 @@ c_compiler_version:    # [linux]
   - 9.4                # [linux]
 cxx_compiler_version:  # [linux]
   - 9.4                # [linux]
+# for FFMPEG c11 macros
+c_stdlib:
+  - sysroot            # [linux]
+c_stdlib_version:      # [linux]
+  - 2.17               # [linux]

conda/dali_native_libs/recipe/build.sh

@@ -114,11 +114,11 @@ fname_with_sha256() {
 }
 
 DEPS_LIST=(
-    "$PREFIX/lib/libavformat.so.60"
-    "$PREFIX/lib/libavcodec.so.60"
-    "$PREFIX/lib/libavfilter.so.9"
-    "$PREFIX/lib/libavutil.so.58"
-    "$PREFIX/lib/libswscale.so.7"
+    "$PREFIX/lib/libavformat.so.61"
+    "$PREFIX/lib/libavcodec.so.61"
+    "$PREFIX/lib/libavfilter.so.10"
+    "$PREFIX/lib/libavutil.so.59"
+    "$PREFIX/lib/libswscale.so.8"
     "lib/libcvcuda.so.0"
     "lib/libnvcv_types.so.0"
 )

conda/dali_native_libs/recipe/meta.yaml

@@ -65,13 +65,13 @@ requirements:
     - {{ compiler('c') }}
     - {{ compiler('cxx') }}
     - pkg-config
-    - cmake >=3.12.4
+    - cmake >=3.18
     - make
     - patchelf
     - python-clang
     - git-lfs
   host:
-    - protobuf >=3.6.1
+    - protobuf >=3.26.1
     - libprotobuf-static >=3.6.1
     - libjpeg-turbo
     - dali-opencv
@@ -101,7 +101,7 @@ requirements:
     # dependency to install the right version on the libabseil (as protobuf depends on
     # libprotobuf-static and a newer version of libprotobuf-static may be available than
     # the protobuf was build with)
-    - protobuf >=3.6.1
+    - protobuf >=3.26.1
     - cfitsio
     - nvidia-nvimagecodec-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }}
 

conda/dali_python_bindings/recipe/meta.yaml

@@ -64,25 +64,25 @@ requirements:
     - {{ compiler('c') }}
     - {{ compiler('cxx') }}
     - pkg-config
-    - cmake >=3.12.4
+    - cmake >=3.18
     - make
     - patchelf
     - python-clang
     - git-lfs
     - astunparse >=1.6.0
     - gast >=0.3.3
     - dm-tree >=0.1.8
-    - black =24.2.0
-    - black-jupyter =24.2.0
+    - black =24.4.2
+    - black-jupyter =24.4.2
     - bandit
     - nvidia-dali-core{% if environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')|length %}{{"-" + environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')}}{% endif %}-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} ={{ environ.get('DALI_CONDA_BUILD_VERSION', '') }}
   host:
     - dali-opencv
     - dali-ffmpeg
-    - libprotobuf-static >=3.6.1
+    - libprotobuf-static >=3.26.1
     - python
     - future
-    - protobuf >=3.6.1
+    - protobuf >=3.26.1
     - astunparse >=1.6.0
     - gast >=0.3.3
     - dm-tree >=0.1.8

conda/third_party/dali_ffmpeg/recipe/meta.yaml

@@ -12,25 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{% set build_version = "6.1.1" %}
+{% set build_version = "7.0" %}
 
   package:
     name: dali-ffmpeg
     version: {{ build_version }}
 
   source:
-    fn: FFmpeg-n6.1.1.tar.gz
-    url: https://developer.download.nvidia.com/compute/redist/nvidia-dali/FFmpeg-n6.1.1.tar.gz
-    sha256: 7c1ebea95d815e49c1e60c7ee816410dec73a81b8ac002b276780d2f9048e598
+    fn: FFmpeg-n7.0.tar.gz
+    url: https://developer.download.nvidia.com/compute/redist/nvidia-dali/FFmpeg-n7.0.tar.gz
+    sha256: a68fbc06a645cc93e5c877adbaa592cffc40e8595ba50eb716807c4a35bf0f03
 
   build:
     number: 0
     string: dali_ffmpeg
 
   requirements:
     build:
-      - {{ compiler('c') }}
-      - {{ compiler('cxx') }}
+      - {{ compiler("c") }}
+      - {{ compiler("cxx") }}
+      - {{ stdlib("c") }}
       - make
       - nasm
       - libtool

dali/core/util_test.cc

@@ -1,4 +1,4 @@
-// Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+// Copyright (c) 2019, 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -63,4 +63,14 @@ TEST(Volume, Ranges) {
   EXPECT_EQ(volume(b + 2, e - 2), 4*9*2*3);
 }
 
+namespace {
+struct Foo {
+  int foo;
+};
+IMPL_HAS_MEMBER(foo);
+}
+
+static_assert(has_member_foo_v<Foo>);
+static_assert(!has_member_foo_v<int>);
+
 }  // namespace dali

dali/operators/reader/loader/video/frames_decoder.cc

@@ -1,4 +1,4 @@
-// Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 #include <memory>
 #include <iomanip>
 #include "dali/core/error_handling.h"
+#include "dali/core/util.h"
 
 
 namespace dali {
@@ -333,13 +334,26 @@ void FramesDecoder::CountFrames(AvState *av_state) {
   }
 }
 
-bool FramesDecoder::IsFormatSeekable() {
-  if (
-    av_state_->ctx_->iformat->read_seek == nullptr &&
-    av_state_->ctx_->iformat->read_seek2 == nullptr) {
-    return false;
+IMPL_HAS_MEMBER(read_seek);
+IMPL_HAS_MEMBER(read_seek2);
+
+template <typename FormatDesc>
+bool IsFormatSeekableHelper(FormatDesc *iformat) {
+  if constexpr (has_member_read_seek_v<FormatDesc>) {
+    static_assert(has_member_read_seek2_v<FormatDesc>);
+    if (iformat->read_seek == nullptr &&
+        iformat->read_seek2 == nullptr)
+      return false;
+  } else {
+    if (iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH))
+      return false;
   }
+  return true;
+}
 
+bool FramesDecoder::IsFormatSeekable() {
+  if (!IsFormatSeekableHelper(av_state_->ctx_->iformat))
+    return false;
   return av_state_->ctx_->pb->read_seek != nullptr;
 }
 

dali/python/bundle-wheel.sh

@@ -106,11 +106,11 @@ make_wheel_record() {
 DEPS_LIST=(
     "${DEPS_PATH}/lib64/libjpeg.so.62"
     "${DEPS_PATH}/lib/libjpeg.so.62"
-    "${DEPS_PATH}/lib/libavformat.so.60"
-    "${DEPS_PATH}/lib/libavcodec.so.60"
-    "${DEPS_PATH}/lib/libavfilter.so.9"
-    "${DEPS_PATH}/lib/libavutil.so.58"
-    "${DEPS_PATH}/lib/libswscale.so.7"
+    "${DEPS_PATH}/lib/libavformat.so.61"
+    "${DEPS_PATH}/lib/libavcodec.so.61"
+    "${DEPS_PATH}/lib/libavfilter.so.10"
+    "${DEPS_PATH}/lib/libavutil.so.59"
+    "${DEPS_PATH}/lib/libswscale.so.8"
     "${DEPS_PATH}/lib/libtiff.so.6"
     "${DEPS_PATH}/lib/libsndfile.so.1"
     "${DEPS_PATH}/lib/libFLAC.so.12"

docker/Dockerfile

@@ -20,7 +20,7 @@ ENV PYVER=${PYVER} \
 RUN ln -s /opt/python/cp${PYV}* /opt/python/v
 
 # install Python bindings and patch it to use the clang we have here
-RUN pip install future setuptools wheel clang==14.0 flake8 bandit astunparse gast dm-tree "black[jupyter]"==24.2.0 && \
+RUN pip install future setuptools wheel clang==14.0 flake8 bandit astunparse gast dm-tree "black[jupyter]"==24.4.2 && \
     PY_CLANG_PATH=$(echo $(pip show clang) | sed 's/.*Location: \(.*\) Requires.*/\1/')/clang/cindex.py && \
     LIBCLANG_PATH=/usr/local/lib/libclang.so && \
     sed -i "s|library_file = None|library_file = \"${LIBCLANG_PATH}\"|" ${PY_CLANG_PATH} && \

docker/Dockerfile.build.aarch64-linux

@@ -37,6 +37,8 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/
     apt-get install -y cuda-cudart-cross-aarch64-${CUDA_CROSS_VERSION} \
                        libcufft-cross-aarch64-${CUDA_CROSS_VERSION} \
                        libcurand-cross-aarch64-${CUDA_CROSS_VERSION} \
+                       libcusolver-cross-aarch64-${CUDA_CROSS_VERSION} \
+                       libcublas-cross-aarch64-${CUDA_CROSS_VERSION} \
                        cuda-driver-cross-aarch64-${CUDA_CROSS_VERSION} \
                        cuda-cccl-cross-aarch64-${CUDA_CROSS_VERSION} \
                        cuda-nvcc-cross-aarch64-${CUDA_CROSS_VERSION} \
@@ -50,7 +52,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/
         curl -O https://bootstrap.pypa.io/get-pip.py; \
     fi && python3 get-pip.py && rm get-pip.py && \
     # decouple libclang and clang installation so libclang changes are not overriden by clang
-    pip install clang==14.0 && pip install libclang==14.0.1 flake8 bandit "black[jupyter]"==24.2.0 && \
+    pip install clang==14.0 && pip install libclang==14.0.1 flake8 bandit "black[jupyter]"==24.4.2 && \
     rm -rf /root/.cache/pip/ && \
     cd /tmp && git clone https://github.com/NixOS/patchelf && cd patchelf && \
     ./bootstrap.sh && ./configure --prefix=/usr/ && make -j install && cd / && rm -rf /tmp/patchelf && \

docker/Dockerfile.deps

@@ -8,7 +8,7 @@ FROM ${BUILDER_EXTRA_DEPS} as extra_deps
 FROM ${FROM_IMAGE_NAME}
 
 # Install yum Dependencies
-RUN yum install -y wget nasm doxygen graphviz gettext xz openssl autogen zip \
+RUN yum install -y wget doxygen graphviz gettext xz openssl autogen zip \
                    devtoolset-10-libasan-devel devtoolset-10-liblsan-devel \
                    devtoolset-10-libtsan-devel devtoolset-10-libubsan-devel \
                    perl perl-IPC-Cmd
@@ -25,9 +25,19 @@ RUN CMAKE_VERSION=3.20.1 && CMAKE_ARCH=$(uname -m) && \
     ./cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh --prefix=/usr/local --skip-license && \
     rm -rf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh
 
+# We need newer NASM than manylinux2014 offers
+RUN cd /tmp && \
+    curl -O -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2  && \
+    tar xjvf nasm-2.15.05.tar.bz2 && \
+    cd nasm-2.15.05 && \
+    ./autogen.sh && \
+    ./configure && \
+    make && \
+    make install
+
 # Clang, build it before deps as deps changes more frequently
 RUN CLANG_VERSION=15.0.2                                                                           && \
-    cd tmp                                                                                         && \
+    cd /tmp                                                                                         && \
     wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-${CLANG_VERSION}.tar.gz    && \
     tar -xf llvmorg-*.tar.gz                                                                       && \
     rm -rf llvmorg-*.tar.gz                                                                        && \

include/dali/core/util.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -300,6 +300,42 @@ struct identity {
   }
 };
 
+/** Implements a trait that checks for the presence of a member called <member_name>
+ *
+ * Usage:
+ * ```
+ * // at namespace/global scope
+ * IMPL_HAS_MEMBER(foo);
+ *
+ * struct S {
+ *     int foo;
+ * };
+ *
+ * template <typename X>
+ * auto foo_or_zero(X x) {
+ *   if constexpr (has_member_foo_v<X>)
+ *     return x.foo;
+ *   else
+ *     return 0;  // no foo in x
+ * }
+ *
+ * int main() {
+ *   S s { 42 };
+ *   cout << foo_or_zero(s) << endl;  // 42
+ *   cout << foo_or_zero(1.234) << endl;  // 0
+ * }
+ *
+ * ```
+ */
+#define IMPL_HAS_MEMBER(member_name)\
+template <typename T, typename = decltype(std::declval<T>().member_name)>\
+std::true_type HasMember_##member_name(T *);\
+std::false_type HasMember_##member_name(...);\
+template <typename T>\
+using has_member_##member_name = \
+  decltype(HasMember_##member_name(std::declval<std::add_pointer_t<T>>()));\
+template <typename T>\
+constexpr bool has_member_##member_name##_v = has_member_##member_name<T>::value
 
 #define IMPL_HAS_NESTED_TYPE(type_name)\
 template <typename T>\

third_party/README.rst

@@ -7,21 +7,23 @@ This part of the repository contains extra dependencies required to build DALI,
 +==================================================================+===========================================================================================================================+===================================================================================================================+
 | `Google Benchmark <https://github.com/google/benchmark>`_        | `1.8.3 <https://github.com/google/benchmark/releases/tag/v1.8.3>`_                                                        | `Apache License 2.0 <https://github.com/google/benchmark/blob/master/LICENSE>`_                                   |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
-| `Boost Preprocessor <https://github.com/boostorg/preprocessor>`_ | `1.84.0 <https://github.com/boostorg/preprocessor/releases/tag/boost-1.84.0>`_                                            | `Boost Software License 1.0 <https://github.com/boostorg/boost/blob/master/LICENSE_1_0.txt>`_                     |
+| `Boost Preprocessor <https://github.com/boostorg/preprocessor>`_ | `1.85.0 <https://github.com/boostorg/preprocessor/releases/tag/boost-1.85.0>`_                                            | `Boost Software License 1.0 <https://github.com/boostorg/boost/blob/master/LICENSE_1_0.txt>`_                     |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
 | `COCO API <https://github.com/cocodataset/cocoapi>`_             | `Top-of-tree (Feb 20, 2020) <https://github.com/cocodataset/cocoapi/tree/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9>`_      | `BSD 2-Clause License <https://github.com/cocodataset/cocoapi/blob/master/license.txt>`_                          |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
 | `CUTLASS <https://github.com/NVIDIA/cutlass>`_                   | `3.4.1 <https://github.com/NVIDIA/cutlass/tree/v3.4.1>`_                                                                  | `BSD 3-Clause License <https://github.com/NVIDIA/cutlass/blob/master/LICENSE.txt>`_                               |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
+| `CV-CUDA <https://github.com/CVCUDA/CV-CUDA>`_                   | `0.7.0 beta <https://github.com/CVCUDA/CV-CUDA/releases/tag/v0.7.0-beta>`_                                                | `Apache License 2.0 <https://github.com/CVCUDA/CV-CUDA/blob/main/LICENSE.md>`_                                    |
++------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
 | `DLPack <https://github.com/dmlc/dlpack>`_                       | `0.8 <https://github.com/dmlc/dlpack/releases/tag/v0.8>`_                                                                 | `Apache License 2.0 <https://github.com/dmlc/dlpack/blob/main/LICENSE>`_                                          |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
 | `FFTS <https://github.com/JanuszL/ffts>`_                        | `Custom fork top-of-tree (Jan 23, 2020) <https://github.com/JanuszL/ffts/tree/c9a9f61a60505751cac385ed062ce2720bdf07d4>`_ | `BSD 3-Clause License <https://github.com/JanuszL/ffts/blob/master/COPYRIGHT>`_                                   |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
 | `GoogleTest <https://github.com/google/googletest>`_             | `1.14.0 <https://github.com/google/googletest/releases/tag/v1.14.0>`_                                                     | `BSD 3-Clause License <https://github.com/google/googletest/blob/master/LICENSE>`_                                |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
-| `pybind11 <https://github.com/pybind/pybind11>`_                 | `2.11.1 <https://github.com/pybind/pybind11/releases/tag/v2.11.1>`_                                                       | `BSD 3-Clause License <https://github.com/pybind/pybind11/blob/master/LICENSE>`_                                  |
+| `pybind11 <https://github.com/pybind/pybind11>`_                 | `2.12.0 <https://github.com/pybind/pybind11/releases/tag/v2.12.0>`_                                                       | `BSD 3-Clause License <https://github.com/pybind/pybind11/blob/master/LICENSE>`_                                  |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
-| `RapidJSON <https://github.com/Tencent/rapidjson>`_              | `Top-of-tree (Feb 9, 2024) <https://github.com/Tencent/rapidjson/commit/3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d>`_       | `MIT License, BSD 3-Clause License, JSON License <https://github.com/Tencent/rapidjson/blob/master/license.txt>`_ |
+| `RapidJSON <https://github.com/Tencent/rapidjson>`_              | `Top-of-tree (Apr 9, 2024) <https://github.com/Tencent/rapidjson/commit/3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d>`_       | `MIT License, BSD 3-Clause License, JSON License <https://github.com/Tencent/rapidjson/blob/master/license.txt>`_ |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+
-| `black <https://github.com/psf/black>`_                          | `24.2.0 <https://pypi.org/project/black/24.2.0/>`_                                                                        | `MIT License <https://github.com/psf/black/blob/main/LICENSE>`_                                                   |
+| `black <https://github.com/psf/black>`_                          | `24.4.2 <https://pypi.org/project/black/24.4.2/>`_                                                                        | `MIT License <https://github.com/psf/black/blob/main/LICENSE>`_                                                   |
 +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+