diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b8f40fb92b2..31f3a50a7df 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install flake8 bandit "black[jupyter]"==24.2.0 + - run: pip install flake8 bandit "black[jupyter]"==24.4.2 - run: black --check --verbose ${{ env.PYTHON_SECURITY_LINT_PATHS }} ${{ env.PYTHON_LINT_PATHS }} ${{ env.AUTOGRAPH_LINT_PATHS }} - run: flake8 --config=.flake8 ${{ env.PYTHON_SECURITY_LINT_PATHS }} ${{ env.PYTHON_LINT_PATHS }} - run: flake8 --config=.flake8.ag ${{ env.AUTOGRAPH_LINT_PATHS }} diff --git a/DALI_DEPS_VERSION b/DALI_DEPS_VERSION index bf35a0d4524..97034b135e4 100644 --- a/DALI_DEPS_VERSION +++ b/DALI_DEPS_VERSION @@ -1 +1 @@ -f0fe9cb92317a788b7dd9c01d73f79bb6aace349 +22cb369c551d2ba5fd1cfe2b667965d968258cda diff --git a/conda/config/conda_build_config.yaml b/conda/config/conda_build_config.yaml index 6347aa5575d..bf1932c8b29 100644 --- a/conda/config/conda_build_config.yaml +++ b/conda/config/conda_build_config.yaml @@ -2,3 +2,8 @@ c_compiler_version: # [linux] - 9.4 # [linux] cxx_compiler_version: # [linux] - 9.4 # [linux] +# for FFMPEG c11 macros +c_stdlib: + - sysroot # [linux] +c_stdlib_version: # [linux] + - 2.17 # [linux] diff --git a/conda/dali_native_libs/recipe/build.sh b/conda/dali_native_libs/recipe/build.sh index 60556dbd33e..30c87349aec 100644 --- a/conda/dali_native_libs/recipe/build.sh +++ b/conda/dali_native_libs/recipe/build.sh @@ -114,11 +114,11 @@ fname_with_sha256() { } DEPS_LIST=( - "$PREFIX/lib/libavformat.so.60" - "$PREFIX/lib/libavcodec.so.60" - "$PREFIX/lib/libavfilter.so.9" - "$PREFIX/lib/libavutil.so.58" - "$PREFIX/lib/libswscale.so.7" + "$PREFIX/lib/libavformat.so.61" + "$PREFIX/lib/libavcodec.so.61" + "$PREFIX/lib/libavfilter.so.10" + "$PREFIX/lib/libavutil.so.59" + "$PREFIX/lib/libswscale.so.8" "lib/libcvcuda.so.0" "lib/libnvcv_types.so.0" ) diff --git a/conda/dali_native_libs/recipe/meta.yaml b/conda/dali_native_libs/recipe/meta.yaml index 3c8a8141df5..92c4a36a867 100644 --- a/conda/dali_native_libs/recipe/meta.yaml +++ b/conda/dali_native_libs/recipe/meta.yaml @@ -65,13 +65,13 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} - pkg-config - - cmake >=3.12.4 + - cmake >=3.18 - make - patchelf - python-clang - git-lfs host: - - protobuf >=3.6.1 + - protobuf >=3.26.1 - libprotobuf-static >=3.6.1 - libjpeg-turbo - dali-opencv @@ -101,7 +101,7 @@ requirements: # dependency to install the right version on the libabseil (as protobuf depends on # libprotobuf-static and a newer version of libprotobuf-static may be available than # the protobuf was build with) - - protobuf >=3.6.1 + - protobuf >=3.26.1 - cfitsio - nvidia-nvimagecodec-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} diff --git a/conda/dali_python_bindings/recipe/meta.yaml b/conda/dali_python_bindings/recipe/meta.yaml index cf2b40ba32f..49440901c15 100644 --- a/conda/dali_python_bindings/recipe/meta.yaml +++ b/conda/dali_python_bindings/recipe/meta.yaml @@ -64,7 +64,7 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} - pkg-config - - cmake >=3.12.4 + - cmake >=3.18 - make - patchelf - python-clang @@ -72,17 +72,17 @@ requirements: - astunparse >=1.6.0 - gast >=0.3.3 - dm-tree >=0.1.8 - - black =24.2.0 - - black-jupyter =24.2.0 + - black =24.4.2 + - black-jupyter =24.4.2 - bandit - nvidia-dali-core{% if environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')|length %}{{"-" + environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')}}{% endif %}-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} ={{ environ.get('DALI_CONDA_BUILD_VERSION', '') }} host: - dali-opencv - dali-ffmpeg - - libprotobuf-static >=3.6.1 + - libprotobuf-static >=3.26.1 - python - future - - protobuf >=3.6.1 + - protobuf >=3.26.1 - astunparse >=1.6.0 - gast >=0.3.3 - dm-tree >=0.1.8 diff --git a/conda/third_party/dali_ffmpeg/recipe/meta.yaml b/conda/third_party/dali_ffmpeg/recipe/meta.yaml index c3df7cafefe..4826e1e0885 100644 --- a/conda/third_party/dali_ffmpeg/recipe/meta.yaml +++ b/conda/third_party/dali_ffmpeg/recipe/meta.yaml @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -{% set build_version = "6.1.1" %} +{% set build_version = "7.0" %} package: name: dali-ffmpeg version: {{ build_version }} source: - fn: FFmpeg-n6.1.1.tar.gz - url: https://developer.download.nvidia.com/compute/redist/nvidia-dali/FFmpeg-n6.1.1.tar.gz - sha256: 7c1ebea95d815e49c1e60c7ee816410dec73a81b8ac002b276780d2f9048e598 + fn: FFmpeg-n7.0.tar.gz + url: https://developer.download.nvidia.com/compute/redist/nvidia-dali/FFmpeg-n7.0.tar.gz + sha256: a68fbc06a645cc93e5c877adbaa592cffc40e8595ba50eb716807c4a35bf0f03 build: number: 0 @@ -29,8 +29,9 @@ requirements: build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} + - {{ compiler("c") }} + - {{ compiler("cxx") }} + - {{ stdlib("c") }} - make - nasm - libtool diff --git a/dali/core/util_test.cc b/dali/core/util_test.cc index 095bfa158df..7e9ecbcf57f 100644 --- a/dali/core/util_test.cc +++ b/dali/core/util_test.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +// Copyright (c) 2019, 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -63,4 +63,14 @@ TEST(Volume, Ranges) { EXPECT_EQ(volume(b + 2, e - 2), 4*9*2*3); } +namespace { +struct Foo { + int foo; +}; +IMPL_HAS_MEMBER(foo); +} + +static_assert(has_member_foo_v); +static_assert(!has_member_foo_v); + } // namespace dali diff --git a/dali/operators/reader/loader/video/frames_decoder.cc b/dali/operators/reader/loader/video/frames_decoder.cc index 7aa3acd39e5..b9c7fbc22f0 100644 --- a/dali/operators/reader/loader/video/frames_decoder.cc +++ b/dali/operators/reader/loader/video/frames_decoder.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include #include "dali/core/error_handling.h" +#include "dali/core/util.h" namespace dali { @@ -333,13 +334,26 @@ void FramesDecoder::CountFrames(AvState *av_state) { } } -bool FramesDecoder::IsFormatSeekable() { - if ( - av_state_->ctx_->iformat->read_seek == nullptr && - av_state_->ctx_->iformat->read_seek2 == nullptr) { - return false; +IMPL_HAS_MEMBER(read_seek); +IMPL_HAS_MEMBER(read_seek2); + +template +bool IsFormatSeekableHelper(FormatDesc *iformat) { + if constexpr (has_member_read_seek_v) { + static_assert(has_member_read_seek2_v); + if (iformat->read_seek == nullptr && + iformat->read_seek2 == nullptr) + return false; + } else { + if (iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH)) + return false; } + return true; +} +bool FramesDecoder::IsFormatSeekable() { + if (!IsFormatSeekableHelper(av_state_->ctx_->iformat)) + return false; return av_state_->ctx_->pb->read_seek != nullptr; } diff --git a/dali/python/bundle-wheel.sh b/dali/python/bundle-wheel.sh index 4768d3648af..c72378d617b 100755 --- a/dali/python/bundle-wheel.sh +++ b/dali/python/bundle-wheel.sh @@ -106,11 +106,11 @@ make_wheel_record() { DEPS_LIST=( "${DEPS_PATH}/lib64/libjpeg.so.62" "${DEPS_PATH}/lib/libjpeg.so.62" - "${DEPS_PATH}/lib/libavformat.so.60" - "${DEPS_PATH}/lib/libavcodec.so.60" - "${DEPS_PATH}/lib/libavfilter.so.9" - "${DEPS_PATH}/lib/libavutil.so.58" - "${DEPS_PATH}/lib/libswscale.so.7" + "${DEPS_PATH}/lib/libavformat.so.61" + "${DEPS_PATH}/lib/libavcodec.so.61" + "${DEPS_PATH}/lib/libavfilter.so.10" + "${DEPS_PATH}/lib/libavutil.so.59" + "${DEPS_PATH}/lib/libswscale.so.8" "${DEPS_PATH}/lib/libtiff.so.6" "${DEPS_PATH}/lib/libsndfile.so.1" "${DEPS_PATH}/lib/libFLAC.so.12" diff --git a/docker/Dockerfile b/docker/Dockerfile index 666b610b8b7..d6f8ee6ae4c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ ENV PYVER=${PYVER} \ RUN ln -s /opt/python/cp${PYV}* /opt/python/v # install Python bindings and patch it to use the clang we have here -RUN pip install future setuptools wheel clang==14.0 flake8 bandit astunparse gast dm-tree "black[jupyter]"==24.2.0 && \ +RUN pip install future setuptools wheel clang==14.0 flake8 bandit astunparse gast dm-tree "black[jupyter]"==24.4.2 && \ PY_CLANG_PATH=$(echo $(pip show clang) | sed 's/.*Location: \(.*\) Requires.*/\1/')/clang/cindex.py && \ LIBCLANG_PATH=/usr/local/lib/libclang.so && \ sed -i "s|library_file = None|library_file = \"${LIBCLANG_PATH}\"|" ${PY_CLANG_PATH} && \ diff --git a/docker/Dockerfile.build.aarch64-linux b/docker/Dockerfile.build.aarch64-linux index e9f93b9ca78..fc4b07dac07 100644 --- a/docker/Dockerfile.build.aarch64-linux +++ b/docker/Dockerfile.build.aarch64-linux @@ -37,6 +37,8 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ apt-get install -y cuda-cudart-cross-aarch64-${CUDA_CROSS_VERSION} \ libcufft-cross-aarch64-${CUDA_CROSS_VERSION} \ libcurand-cross-aarch64-${CUDA_CROSS_VERSION} \ + libcusolver-cross-aarch64-${CUDA_CROSS_VERSION} \ + libcublas-cross-aarch64-${CUDA_CROSS_VERSION} \ cuda-driver-cross-aarch64-${CUDA_CROSS_VERSION} \ cuda-cccl-cross-aarch64-${CUDA_CROSS_VERSION} \ cuda-nvcc-cross-aarch64-${CUDA_CROSS_VERSION} \ @@ -50,7 +52,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ curl -O https://bootstrap.pypa.io/get-pip.py; \ fi && python3 get-pip.py && rm get-pip.py && \ # decouple libclang and clang installation so libclang changes are not overriden by clang - pip install clang==14.0 && pip install libclang==14.0.1 flake8 bandit "black[jupyter]"==24.2.0 && \ + pip install clang==14.0 && pip install libclang==14.0.1 flake8 bandit "black[jupyter]"==24.4.2 && \ rm -rf /root/.cache/pip/ && \ cd /tmp && git clone https://github.com/NixOS/patchelf && cd patchelf && \ ./bootstrap.sh && ./configure --prefix=/usr/ && make -j install && cd / && rm -rf /tmp/patchelf && \ diff --git a/docker/Dockerfile.deps b/docker/Dockerfile.deps index 8d4d96da913..d5e2ba3c403 100644 --- a/docker/Dockerfile.deps +++ b/docker/Dockerfile.deps @@ -8,7 +8,7 @@ FROM ${BUILDER_EXTRA_DEPS} as extra_deps FROM ${FROM_IMAGE_NAME} # Install yum Dependencies -RUN yum install -y wget nasm doxygen graphviz gettext xz openssl autogen zip \ +RUN yum install -y wget doxygen graphviz gettext xz openssl autogen zip \ devtoolset-10-libasan-devel devtoolset-10-liblsan-devel \ devtoolset-10-libtsan-devel devtoolset-10-libubsan-devel \ perl perl-IPC-Cmd @@ -25,9 +25,19 @@ RUN CMAKE_VERSION=3.20.1 && CMAKE_ARCH=$(uname -m) && \ ./cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh --prefix=/usr/local --skip-license && \ rm -rf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.sh +# We need newer NASM than manylinux2014 offers +RUN cd /tmp && \ + curl -O -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && \ + tar xjvf nasm-2.15.05.tar.bz2 && \ + cd nasm-2.15.05 && \ + ./autogen.sh && \ + ./configure && \ + make && \ + make install + # Clang, build it before deps as deps changes more frequently RUN CLANG_VERSION=15.0.2 && \ - cd tmp && \ + cd /tmp && \ wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-${CLANG_VERSION}.tar.gz && \ tar -xf llvmorg-*.tar.gz && \ rm -rf llvmorg-*.tar.gz && \ diff --git a/include/dali/core/util.h b/include/dali/core/util.h index 158b5c1f496..866a7934ad2 100644 --- a/include/dali/core/util.h +++ b/include/dali/core/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -300,6 +300,42 @@ struct identity { } }; +/** Implements a trait that checks for the presence of a member called + * + * Usage: + * ``` + * // at namespace/global scope + * IMPL_HAS_MEMBER(foo); + * + * struct S { + * int foo; + * }; + * + * template + * auto foo_or_zero(X x) { + * if constexpr (has_member_foo_v) + * return x.foo; + * else + * return 0; // no foo in x + * } + * + * int main() { + * S s { 42 }; + * cout << foo_or_zero(s) << endl; // 42 + * cout << foo_or_zero(1.234) << endl; // 0 + * } + * + * ``` + */ +#define IMPL_HAS_MEMBER(member_name)\ +template ().member_name)>\ +std::true_type HasMember_##member_name(T *);\ +std::false_type HasMember_##member_name(...);\ +template \ +using has_member_##member_name = \ + decltype(HasMember_##member_name(std::declval>()));\ +template \ +constexpr bool has_member_##member_name##_v = has_member_##member_name::value #define IMPL_HAS_NESTED_TYPE(type_name)\ template \ diff --git a/third_party/README.rst b/third_party/README.rst index 39c9f143261..1a02b92d7f4 100644 --- a/third_party/README.rst +++ b/third_party/README.rst @@ -7,21 +7,23 @@ This part of the repository contains extra dependencies required to build DALI, +==================================================================+===========================================================================================================================+===================================================================================================================+ | `Google Benchmark `_ | `1.8.3 `_ | `Apache License 2.0 `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `Boost Preprocessor `_ | `1.84.0 `_ | `Boost Software License 1.0 `_ | +| `Boost Preprocessor `_ | `1.85.0 `_ | `Boost Software License 1.0 `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `COCO API `_ | `Top-of-tree (Feb 20, 2020) `_ | `BSD 2-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `CUTLASS `_ | `3.4.1 `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ +| `CV-CUDA `_ | `0.7.0 beta `_ | `Apache License 2.0 `_ | ++------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `DLPack `_ | `0.8 `_ | `Apache License 2.0 `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `FFTS `_ | `Custom fork top-of-tree (Jan 23, 2020) `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `GoogleTest `_ | `1.14.0 `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `pybind11 `_ | `2.11.1 `_ | `BSD 3-Clause License `_ | +| `pybind11 `_ | `2.12.0 `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `RapidJSON `_ | `Top-of-tree (Feb 9, 2024) `_ | `MIT License, BSD 3-Clause License, JSON License `_ | +| `RapidJSON `_ | `Top-of-tree (Apr 9, 2024) `_ | `MIT License, BSD 3-Clause License, JSON License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `black `_ | `24.2.0 `_ | `MIT License `_ | +| `black `_ | `24.4.2 `_ | `MIT License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ diff --git a/third_party/boost/preprocessor b/third_party/boost/preprocessor index 667e87b3392..c4ea7e40d36 160000 --- a/third_party/boost/preprocessor +++ b/third_party/boost/preprocessor @@ -1 +1 @@ -Subproject commit 667e87b3392db338a919cbe0213979713aca52e3 +Subproject commit c4ea7e40d365ba28faecef8917d5c3f1e0121bf9 diff --git a/third_party/cvcuda b/third_party/cvcuda index f5e4f6b480f..11d40a4e1f2 160000 --- a/third_party/cvcuda +++ b/third_party/cvcuda @@ -1 +1 @@ -Subproject commit f5e4f6b480fb4239712a99d53b2c437e84bfc671 +Subproject commit 11d40a4e1f201dff9e808a0cb1b0da466345cb07 diff --git a/third_party/pybind11 b/third_party/pybind11 index 8a099e44b3d..3e9dfa28669 160000 --- a/third_party/pybind11 +++ b/third_party/pybind11 @@ -1 +1 @@ -Subproject commit 8a099e44b3d5f85b20f05828d919d2332a8de841 +Subproject commit 3e9dfa2866941655c56877882565e7577de6fc7b diff --git a/third_party/rapidjson b/third_party/rapidjson index 3f73edae00a..ab1842a2dae 160000 --- a/third_party/rapidjson +++ b/third_party/rapidjson @@ -1 +1 @@ -Subproject commit 3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d +Subproject commit ab1842a2dae061284c0a62dca1cc6d5e7e37e346