From f579c74f3c20b7bd667b0acf21b9b0d1a2e94eec Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Sat, 23 Mar 2024 11:37:42 -0400 Subject: [PATCH 01/15] rocmPackages: extend rocm-runtime ISA compatibility Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/clr/default.nix | 6 ++++++ .../rocm-modules/6/rocm-runtime/default.nix | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/pkgs/development/rocm-modules/6/clr/default.nix b/pkgs/development/rocm-modules/6/clr/default.nix index c5a3f641ff41a..f2a0ea5c5d44d 100644 --- a/pkgs/development/rocm-modules/6/clr/default.nix +++ b/pkgs/development/rocm-modules/6/clr/default.nix @@ -111,6 +111,8 @@ in stdenv.mkDerivation (finalAttrs: { url = "https://github.com/ROCm/clr/commit/77c581a3ebd47b5e2908973b70adea66891159ee.patch"; hash = "sha256-auBedbd7rghlKav7A9V6l64J7VmtE9GizIdi5gWj+fs="; }) + ./0001-Improve-hipamd-compat-check.patch + ./0001-improve-rocclr-isa-compatibility-check.patch ]; postPatch = '' @@ -124,6 +126,10 @@ in stdenv.mkDerivation (finalAttrs: { substituteInPlace hipamd/src/hip_embed_pch.sh \ --replace "\''$LLVM_DIR/bin/clang" "${clang}/bin/clang" + + # https://lists.debian.org/debian-ai/2024/02/msg00178.html + substituteInPlace rocclr/utils/flags.hpp \ + --replace-fail "HIP_USE_RUNTIME_UNBUNDLER, false" "HIP_USE_RUNTIME_UNBUNDLER, true" ''; postInstall = '' diff --git a/pkgs/development/rocm-modules/6/rocm-runtime/default.nix b/pkgs/development/rocm-modules/6/rocm-runtime/default.nix index 8c3d0cdc976db..f603b708265de 100644 --- a/pkgs/development/rocm-modules/6/rocm-runtime/default.nix +++ b/pkgs/development/rocm-modules/6/rocm-runtime/default.nix @@ -1,6 +1,7 @@ { lib , stdenv , fetchFromGitHub +, fetchpatch , rocmUpdateScript , pkg-config , cmake @@ -42,6 +43,15 @@ stdenv.mkDerivation (finalAttrs: { libxml2 ]; + patches = [ + (fetchpatch { + name = "extend-isa-compatibility-check.patch"; + url = "https://salsa.debian.org/rocm-team/rocr-runtime/-/raw/076026d43bbee7f816b81fea72f984213a9ff961/debian/patches/0004-extend-isa-compatibility-check.patch"; + hash = "sha256-cC030zVGS4kNXwaztv5cwfXfVwOldpLGV9iYgEfPEnY="; + stripLen = 1; + }) + ]; + postPatch = '' patchShebangs image/blit_src/create_hsaco_ascii_file.sh patchShebangs core/runtime/trap_handler/create_trap_handler_header.sh From f1367529e9c92acb1cc1a431dae39440a045c64f Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Sat, 23 Mar 2024 11:38:02 -0400 Subject: [PATCH 02/15] rocmPackages: extend clr ISA compatibility Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/clr/default.nix | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkgs/development/rocm-modules/6/clr/default.nix b/pkgs/development/rocm-modules/6/clr/default.nix index f2a0ea5c5d44d..3c8d20fb393ae 100644 --- a/pkgs/development/rocm-modules/6/clr/default.nix +++ b/pkgs/development/rocm-modules/6/clr/default.nix @@ -111,8 +111,16 @@ in stdenv.mkDerivation (finalAttrs: { url = "https://github.com/ROCm/clr/commit/77c581a3ebd47b5e2908973b70adea66891159ee.patch"; hash = "sha256-auBedbd7rghlKav7A9V6l64J7VmtE9GizIdi5gWj+fs="; }) - ./0001-Improve-hipamd-compat-check.patch - ./0001-improve-rocclr-isa-compatibility-check.patch + (fetchpatch { + name = "Improve-hipamd-compat-check.patch"; + url = "https://github.com/GZGavinZhao/clr/commit/f52172a0767f88bf386dc615a3354156d023bdb8.patch"; + hash = "sha256-kbEeJsQgAxbNfBCPB2Jny4z526UAnIxHNQLGsD2iFvg="; + }) + (fetchpatch { + name = "improve-rocclr-isa-compatibility-check.patch"; + url = "https://github.com/GZGavinZhao/clr/commit/2783c57b0f225ad8bc553e2d244837d57d8375bc.patch"; + hash = "sha256-uQMex3gT/LqgQeMETbldpVODWorGLD/YMgjdwHlhd+M="; + }) ]; postPatch = '' From d6adc4b1d41559e3d46c88cd0fd941f9f4f063f3 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Sat, 23 Mar 2024 11:42:10 -0400 Subject: [PATCH 03/15] rocmPackages: extend tensile ISA compatibility Signed-off-by: Gavin Zhao --- .../rocm-modules/6/tensile/default.nix | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkgs/development/rocm-modules/6/tensile/default.nix b/pkgs/development/rocm-modules/6/tensile/default.nix index af33fd789e846..410264a235ee1 100644 --- a/pkgs/development/rocm-modules/6/tensile/default.nix +++ b/pkgs/development/rocm-modules/6/tensile/default.nix @@ -1,6 +1,7 @@ { lib , stdenv , fetchFromGitHub +, fetchpatch , rocmUpdateScript , buildPythonPackage , pytestCheckHook @@ -34,6 +35,14 @@ buildPythonPackage rec { joblib ]; + patches = [ + (fetchpatch { + name = "Extend-Tensile-HIP-ISA-compatibility.patch"; + url = "https://github.com/GZGavinZhao/Tensile/commit/855cb15839849addb0816a6dde45772034a3e41f.patch"; + hash = "sha256-d+fVf/vz+sxGqJ96vuxe0jRMgbC5K6j5FQ5SJ1e3Sl8="; + }) + ]; + doCheck = false; # Too many errors, not sure how to set this up properly nativeCheckInputs = [ @@ -42,9 +51,9 @@ buildPythonPackage rec { rocminfo ]; - preCheck = '' - export ROCM_PATH=${rocminfo} - ''; + env = { + ROCM_PATH = rocminfo; + }; pythonImportsCheck = [ "Tensile" ]; From 41f946176f12adcc897779c530e4abc9bcf396d4 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Sat, 23 Mar 2024 11:44:21 -0400 Subject: [PATCH 04/15] rocmPackages: extend rocblas ISA compatibility Signed-off-by: Gavin Zhao --- .../rocm-modules/6/rocblas/default.nix | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pkgs/development/rocm-modules/6/rocblas/default.nix b/pkgs/development/rocm-modules/6/rocblas/default.nix index 296167bb6f287..c5c399777390b 100644 --- a/pkgs/development/rocm-modules/6/rocblas/default.nix +++ b/pkgs/development/rocm-modules/6/rocblas/default.nix @@ -2,6 +2,7 @@ , lib , stdenv , fetchFromGitHub +, fetchpatch , rocmUpdateScript , runCommand , cmake @@ -21,8 +22,13 @@ , buildBenchmarks ? false , tensileLogic ? "asm_full" , tensileCOVersion ? "default" -, tensileSepArch ? true -, tensileLazyLib ? true +# https://github.com/ROCm/Tensile/issues/1757 +# Allows gfx101* users to use rocBLAS normally. +# Turn the below two values to `true` after the fix has been cherry-picked +# into a release. Just backporting that single fix is not enough because it +# depends on some previous commits. +, tensileSepArch ? false +, tensileLazyLib ? false , tensileLibFormat ? "msgpack" , gpuTargets ? [ "all" ] }: @@ -141,6 +147,14 @@ in stdenv.mkDerivation (finalAttrs: { "-DCMAKE_CXX_FLAGS=-I${amd-blis}/include/blis" ]; + patches = [ + (fetchpatch { + name = "Extend-rocBLAS-HIP-ISA-compatibility.patch"; + url = "https://github.com/GZGavinZhao/rocBLAS/commit/89b75ff9cc731f71f370fad90517395e117b03bb.patch"; + hash = "sha256-W/ohOOyNCcYYLOiQlPzsrTlNtCBdJpKVxO8s+4G7sjo="; + }) + ]; + postPatch = lib.optionalString (finalAttrs.pname != "rocblas") '' # Return early and install tensile files manually substituteInPlace library/src/CMakeLists.txt \ From e24032988f4641951e6ec635e8b7062a5dcff01f Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Sat, 23 Mar 2024 11:54:06 -0400 Subject: [PATCH 05/15] rocmPackages: extend miopen ISA compatibility Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/miopen/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/development/rocm-modules/6/miopen/default.nix b/pkgs/development/rocm-modules/6/miopen/default.nix index 09bcf10429530..0b57cf6592cbf 100644 --- a/pkgs/development/rocm-modules/6/miopen/default.nix +++ b/pkgs/development/rocm-modules/6/miopen/default.nix @@ -116,6 +116,11 @@ in stdenv.mkDerivation (finalAttrs: { url = "https://github.com/ROCm/MIOpen/commit/3413d2daaeb44b7d6eadcc03033a5954a118491e.patch"; hash = "sha256-ST4snUcTmmSI1Ogx815KEX9GdMnmubsavDzXCGJkiKs="; }) + (fetchpatch { + name = "Extend-MIOpen-ISA-compatibility.patch"; + url = "https://github.com/GZGavinZhao/MIOpen/commit/416088b534618bd669a765afce59cfc7197064c1.patch"; + hash = "sha256-OwONCA68y8s2GqtQj+OtotXwUXQ5jM8tpeM92iaD4MU="; + }) ]; outputs = [ From e15c44956b5cd9d8001096cba220afdb037f2e03 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Sun, 24 Mar 2024 23:58:41 -0400 Subject: [PATCH 06/15] rocmPackages: respect NIX_BUILD_CORES and cleanup CMake options in rocblas Signed-off-by: Gavin Zhao --- .../rocm-modules/6/rocblas/default.nix | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/pkgs/development/rocm-modules/6/rocblas/default.nix b/pkgs/development/rocm-modules/6/rocblas/default.nix index c5c399777390b..ff5b57431b8f8 100644 --- a/pkgs/development/rocm-modules/6/rocblas/default.nix +++ b/pkgs/development/rocm-modules/6/rocblas/default.nix @@ -120,31 +120,31 @@ in stdenv.mkDerivation (finalAttrs: { ]; cmakeFlags = [ - "-DCMAKE_C_COMPILER=hipcc" - "-DCMAKE_CXX_COMPILER=hipcc" - "-Dpython=python3" - "-DAMDGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}" - "-DBUILD_WITH_TENSILE=${if buildTensile then "ON" else "OFF"}" - # Manually define CMAKE_INSTALL_ - # See: https://github.com/NixOS/nixpkgs/pull/197838 - "-DCMAKE_INSTALL_BINDIR=bin" - "-DCMAKE_INSTALL_LIBDIR=lib" - "-DCMAKE_INSTALL_INCLUDEDIR=include" + (lib.cmakeFeature "CMAKE_C_COMPILER" "hipcc") + (lib.cmakeFeature "CMAKE_CXX_COMPILER" "hipcc") + (lib.cmakeFeature "python" "python3") + (lib.cmakeFeature "AMDGPU_TARGETS" (lib.concatStringsSep ";" gpuTargets)) + (lib.cmakeBool "BUILD_WITH_TENSILE" buildTensile) + (lib.cmakeBool "ROCM_SYMLINK_LIBS" false) + # # Manually define CMAKE_INSTALL_ + # # See: https://github.com/NixOS/nixpkgs/pull/197838 + # "-DCMAKE_INSTALL_BINDIR=bin" + # "-DCMAKE_INSTALL_LIBDIR=lib" + # "-DCMAKE_INSTALL_INCLUDEDIR=include" + (lib.cmakeBool "BUILD_CLIENTS_TESTS" buildTests) + (lib.cmakeBool "BUILD_CLIENTS_BENCHMARKS" buildBenchmarks) ] ++ lib.optionals buildTensile [ - "-DVIRTUALENV_HOME_DIR=/build/source/tensile" - "-DTensile_TEST_LOCAL_PATH=/build/source/tensile" - "-DTensile_ROOT=/build/source/tensile/${python3.sitePackages}/Tensile" - "-DTensile_LOGIC=${tensileLogic}" - "-DTensile_CODE_OBJECT_VERSION=${tensileCOVersion}" - "-DTensile_SEPARATE_ARCHITECTURES=${if tensileSepArch then "ON" else "OFF"}" - "-DTensile_LAZY_LIBRARY_LOADING=${if tensileLazyLib then "ON" else "OFF"}" - "-DTensile_LIBRARY_FORMAT=${tensileLibFormat}" - ] ++ lib.optionals buildTests [ - "-DBUILD_CLIENTS_TESTS=ON" - ] ++ lib.optionals buildBenchmarks [ - "-DBUILD_CLIENTS_BENCHMARKS=ON" + (lib.cmakeFeature "VIRTUALENV_HOME_DIR" "/build/source/tensile") + (lib.cmakeFeature "Tensile_TEST_LOCAL_PATH" "/build/source/tensile") + (lib.cmakeFeature "Tensile_ROOT" "/build/source/tensile/${python3.sitePackages}/Tensile") + (lib.cmakeFeature "Tensile_LOGIC" tensileLogic) + (lib.cmakeFeature "Tensile_CODE_OBJECT_VERSION" tensileCOVersion) + (lib.cmakeBool "Tensile_SEPARATE_ARCHITECTURES" tensileSepArch) + (lib.cmakeBool "Tensile_LAZY_LIBRARY_LOADING" tensileLazyLib) + (lib.cmakeFeature "Tensile_LIBRARY_FORMAT" tensileLibFormat) + (lib.cmakeBool "Tensile_PRINT_DEBUG" true) ] ++ lib.optionals (buildTests || buildBenchmarks) [ - "-DCMAKE_CXX_FLAGS=-I${amd-blis}/include/blis" + (lib.cmakeFeature "CMAKE_CXX_FLAGS" "-I${amd-blis}/include/blis") ]; patches = [ @@ -155,7 +155,10 @@ in stdenv.mkDerivation (finalAttrs: { }) ]; - postPatch = lib.optionalString (finalAttrs.pname != "rocblas") '' + postPatch = '' + substituteInPlace cmake/build-options.cmake \ + --replace-fail 'Tensile_CPU_THREADS ""' 'Tensile_CPU_THREADS "$ENV{NIX_BUILD_CORES}"' + '' + lib.optionalString (finalAttrs.pname != "rocblas") '' # Return early and install tensile files manually substituteInPlace library/src/CMakeLists.txt \ --replace "set_target_properties( TensileHost PROPERTIES OUTPUT_NAME" "return()''\nset_target_properties( TensileHost PROPERTIES OUTPUT_NAME" From f4f68f4f6bb8b7c41f4d8fa68f44e0824b068a67 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Thu, 28 Mar 2024 11:16:00 -0400 Subject: [PATCH 07/15] rocmPackages: cleanup and remove double building for rocblas Signed-off-by: Gavin Zhao --- .../rocm-modules/6/rocblas/default.nix | 105 +----------------- .../rocm-modules/6/tensile/default.nix | 5 + 2 files changed, 10 insertions(+), 100 deletions(-) diff --git a/pkgs/development/rocm-modules/6/rocblas/default.nix b/pkgs/development/rocm-modules/6/rocblas/default.nix index ff5b57431b8f8..e6b7ed34717cc 100644 --- a/pkgs/development/rocm-modules/6/rocblas/default.nix +++ b/pkgs/development/rocm-modules/6/rocblas/default.nix @@ -33,51 +33,7 @@ , gpuTargets ? [ "all" ] }: -let - # NOTE: Update the default GPU targets on every update - gfx80 = (rocblas.override { - gpuTargets = [ - "gfx803" - ]; - }).overrideAttrs { pname = "rocblas-tensile-gfx80"; }; - - gfx90 = (rocblas.override { - gpuTargets = [ - "gfx900" - "gfx906:xnack-" - "gfx908:xnack-" - "gfx90a:xnack+" - "gfx90a:xnack-" - ]; - }).overrideAttrs { pname = "rocblas-tensile-gfx90"; }; - - gfx94 = (rocblas.override { - gpuTargets = [ - "gfx940" - "gfx941" - "gfx942" - ]; - }).overrideAttrs { pname = "rocblas-tensile-gfx94"; }; - - gfx10 = (rocblas.override { - gpuTargets = [ - "gfx1010" - "gfx1012" - "gfx1030" - ]; - }).overrideAttrs { pname = "rocblas-tensile-gfx10"; }; - - gfx11 = (rocblas.override { - gpuTargets = [ - "gfx1100" - "gfx1101" - "gfx1102" - ]; - }).overrideAttrs { pname = "rocblas-tensile-gfx11"; }; - - # Unfortunately, we have to do two full builds, otherwise we get overlapping _fallback.dat files - fallbacks = rocblas.overrideAttrs { pname = "rocblas-tensile-fallbacks"; }; -in stdenv.mkDerivation (finalAttrs: { +stdenv.mkDerivation (finalAttrs: { pname = "rocblas"; version = "6.0.2"; @@ -100,6 +56,7 @@ in stdenv.mkDerivation (finalAttrs: { cmake rocm-cmake clr + tensile ]; buildInputs = [ @@ -126,17 +83,11 @@ in stdenv.mkDerivation (finalAttrs: { (lib.cmakeFeature "AMDGPU_TARGETS" (lib.concatStringsSep ";" gpuTargets)) (lib.cmakeBool "BUILD_WITH_TENSILE" buildTensile) (lib.cmakeBool "ROCM_SYMLINK_LIBS" false) - # # Manually define CMAKE_INSTALL_ - # # See: https://github.com/NixOS/nixpkgs/pull/197838 - # "-DCMAKE_INSTALL_BINDIR=bin" - # "-DCMAKE_INSTALL_LIBDIR=lib" - # "-DCMAKE_INSTALL_INCLUDEDIR=include" + (lib.cmakeFeature "ROCBLAS_TENSILE_LIBRARY_DIR" "lib/rocblas") (lib.cmakeBool "BUILD_CLIENTS_TESTS" buildTests) (lib.cmakeBool "BUILD_CLIENTS_BENCHMARKS" buildBenchmarks) ] ++ lib.optionals buildTensile [ - (lib.cmakeFeature "VIRTUALENV_HOME_DIR" "/build/source/tensile") - (lib.cmakeFeature "Tensile_TEST_LOCAL_PATH" "/build/source/tensile") - (lib.cmakeFeature "Tensile_ROOT" "/build/source/tensile/${python3.sitePackages}/Tensile") + (lib.cmakeBool "BUILD_WITH_PIP" false) (lib.cmakeFeature "Tensile_LOGIC" tensileLogic) (lib.cmakeFeature "Tensile_CODE_OBJECT_VERSION" tensileCOVersion) (lib.cmakeBool "Tensile_SEPARATE_ARCHITECTURES" tensileSepArch) @@ -155,56 +106,10 @@ in stdenv.mkDerivation (finalAttrs: { }) ]; + # Pass $NIX_BUILD_CORES to Tensile postPatch = '' substituteInPlace cmake/build-options.cmake \ --replace-fail 'Tensile_CPU_THREADS ""' 'Tensile_CPU_THREADS "$ENV{NIX_BUILD_CORES}"' - '' + lib.optionalString (finalAttrs.pname != "rocblas") '' - # Return early and install tensile files manually - substituteInPlace library/src/CMakeLists.txt \ - --replace "set_target_properties( TensileHost PROPERTIES OUTPUT_NAME" "return()''\nset_target_properties( TensileHost PROPERTIES OUTPUT_NAME" - '' + lib.optionalString (buildTensile && finalAttrs.pname == "rocblas") '' - # Link the prebuilt Tensile files - mkdir -p build/Tensile/library - - for path in ${gfx80} ${gfx90} ${gfx94} ${gfx10} ${gfx11} ${fallbacks}; do - ln -s $path/lib/rocblas/library/* build/Tensile/library - done - - unlink build/Tensile/library/TensileManifest.txt - '' + lib.optionalString buildTensile '' - # Tensile REALLY wants to write to the nix directory if we include it normally - cp -a ${tensile} tensile - chmod +w -R tensile - - # Rewrap Tensile - substituteInPlace tensile/bin/{.t*,.T*,*} \ - --replace "${tensile}" "/build/source/tensile" - - substituteInPlace CMakeLists.txt \ - --replace "include(virtualenv)" "" \ - --replace "virtualenv_install(\''${Tensile_TEST_LOCAL_PATH})" "" - ''; - - postInstall = lib.optionalString (finalAttrs.pname == "rocblas") '' - ln -sf ${fallbacks}/lib/rocblas/library/TensileManifest.txt $out/lib/rocblas/library - '' + lib.optionalString (finalAttrs.pname != "rocblas") '' - mkdir -p $out/lib/rocblas/library - rm -rf $out/share - '' + lib.optionalString (finalAttrs.pname != "rocblas" && finalAttrs.pname != "rocblas-tensile-fallbacks") '' - rm Tensile/library/{TensileManifest.txt,*_fallback.dat} - mv Tensile/library/* $out/lib/rocblas/library - '' + lib.optionalString (finalAttrs.pname == "rocblas-tensile-fallbacks") '' - mv Tensile/library/{TensileManifest.txt,*_fallback.dat} $out/lib/rocblas/library - '' + lib.optionalString buildTests '' - mkdir -p $test/bin - cp -a $out/bin/* $test/bin - rm $test/bin/*-bench || true - '' + lib.optionalString buildBenchmarks '' - mkdir -p $benchmark/bin - cp -a $out/bin/* $benchmark/bin - rm $benchmark/bin/*-test || true - '' + lib.optionalString (buildTests || buildBenchmarks ) '' - rm -rf $out/bin ''; passthru.updateScript = rocmUpdateScript { diff --git a/pkgs/development/rocm-modules/6/tensile/default.nix b/pkgs/development/rocm-modules/6/tensile/default.nix index 410264a235ee1..6df403249e9aa 100644 --- a/pkgs/development/rocm-modules/6/tensile/default.nix +++ b/pkgs/development/rocm-modules/6/tensile/default.nix @@ -41,6 +41,11 @@ buildPythonPackage rec { url = "https://github.com/GZGavinZhao/Tensile/commit/855cb15839849addb0816a6dde45772034a3e41f.patch"; hash = "sha256-d+fVf/vz+sxGqJ96vuxe0jRMgbC5K6j5FQ5SJ1e3Sl8="; }) + (fetchpatch { + name = "Don-t-copy-file-twice-in-copyStaticFiles.patch"; + url = "https://github.com/GZGavinZhao/Tensile/commit/9e14d5a00a096bddac605910a0e4dfb4c35bb0d5.patch"; + hash = "sha256-gOzjJyD1K056OFQ+hK5nbUeBhxLTIgQLoT+0K12SypI="; + }) ]; doCheck = false; # Too many errors, not sure how to set this up properly From d276a67466c27e39ba4d343d6f54006d09446a31 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Thu, 28 Mar 2024 11:45:58 -0400 Subject: [PATCH 08/15] rocmPackages: don't build arches in parallel in rccl Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/rccl/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/development/rocm-modules/6/rccl/default.nix b/pkgs/development/rocm-modules/6/rccl/default.nix index cdbbd70440c20..a830883b4f82e 100644 --- a/pkgs/development/rocm-modules/6/rccl/default.nix +++ b/pkgs/development/rocm-modules/6/rccl/default.nix @@ -65,7 +65,9 @@ stdenv.mkDerivation (finalAttrs: { # Really strange behavior, `#!/usr/bin/env perl` should work... substituteInPlace CMakeLists.txt \ - --replace "\''$ \''${hipify-perl_executable}" "${perl}/bin/perl ${hipify}/bin/hipify-perl" + --replace "\''$ \''${hipify-perl_executable}" "${perl}/bin/perl ${hipify}/bin/hipify-perl" \ + --replace-warn "-parallel-jobs=12" "-parallel-jobs=1" \ + --replace-warn "-parallel-jobs=16" "-parallel-jobs=1" ''; postInstall = lib.optionalString buildTests '' From 73e919d8b76ea437b77512e819ed9c3711193775 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Fri, 29 Mar 2024 10:48:59 -0400 Subject: [PATCH 09/15] rocmPackages: install headers correctly in rocblas Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/rocblas/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/rocm-modules/6/rocblas/default.nix b/pkgs/development/rocm-modules/6/rocblas/default.nix index e6b7ed34717cc..f1c709dc79fc6 100644 --- a/pkgs/development/rocm-modules/6/rocblas/default.nix +++ b/pkgs/development/rocm-modules/6/rocblas/default.nix @@ -86,6 +86,8 @@ stdenv.mkDerivation (finalAttrs: { (lib.cmakeFeature "ROCBLAS_TENSILE_LIBRARY_DIR" "lib/rocblas") (lib.cmakeBool "BUILD_CLIENTS_TESTS" buildTests) (lib.cmakeBool "BUILD_CLIENTS_BENCHMARKS" buildBenchmarks) + # rocblas header files are not installed unless we set this + (lib.cmakeFeature "CMAKE_INSTALL_INCLUDEDIR" "include") ] ++ lib.optionals buildTensile [ (lib.cmakeBool "BUILD_WITH_PIP" false) (lib.cmakeFeature "Tensile_LOGIC" tensileLogic) From 7ff6984fd3e0a063b22c00ca56a7bfe502326cba Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Wed, 3 Apr 2024 01:05:46 -0400 Subject: [PATCH 10/15] rocmPackages: fix dependencies in rocblas Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/rocblas/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/rocm-modules/6/rocblas/default.nix b/pkgs/development/rocm-modules/6/rocblas/default.nix index f1c709dc79fc6..9dfa8636f1201 100644 --- a/pkgs/development/rocm-modules/6/rocblas/default.nix +++ b/pkgs/development/rocm-modules/6/rocblas/default.nix @@ -1,5 +1,4 @@ -{ rocblas -, lib +{ lib , stdenv , fetchFromGitHub , fetchpatch @@ -56,6 +55,7 @@ stdenv.mkDerivation (finalAttrs: { cmake rocm-cmake clr + ] ++ lib.optionals buildTensile [ tensile ]; From e30bae6a69fc77dd5c573d42fc7c8e0490d0b63c Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Wed, 3 Apr 2024 01:23:05 -0400 Subject: [PATCH 11/15] rocmPackages: fetch patches from Debian if possible Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/clr/default.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/development/rocm-modules/6/clr/default.nix b/pkgs/development/rocm-modules/6/clr/default.nix index 3c8d20fb393ae..dedda194c8280 100644 --- a/pkgs/development/rocm-modules/6/clr/default.nix +++ b/pkgs/development/rocm-modules/6/clr/default.nix @@ -112,14 +112,14 @@ in stdenv.mkDerivation (finalAttrs: { hash = "sha256-auBedbd7rghlKav7A9V6l64J7VmtE9GizIdi5gWj+fs="; }) (fetchpatch { - name = "Improve-hipamd-compat-check.patch"; - url = "https://github.com/GZGavinZhao/clr/commit/f52172a0767f88bf386dc615a3354156d023bdb8.patch"; - hash = "sha256-kbEeJsQgAxbNfBCPB2Jny4z526UAnIxHNQLGsD2iFvg="; + name = "extend-hip-isa-compatibility-check.patch"; + url = "https://salsa.debian.org/rocm-team/rocm-hipamd/-/raw/d6d20142c37e1dff820950b16ff8f0523241d935/debian/patches/0026-extend-hip-isa-compatibility-check.patch"; + hash = "sha256-eG0ALZZQLRzD7zJueJFhi2emontmYy6xx8Rsm346nQI="; }) (fetchpatch { name = "improve-rocclr-isa-compatibility-check.patch"; - url = "https://github.com/GZGavinZhao/clr/commit/2783c57b0f225ad8bc553e2d244837d57d8375bc.patch"; - hash = "sha256-uQMex3gT/LqgQeMETbldpVODWorGLD/YMgjdwHlhd+M="; + url = "https://salsa.debian.org/rocm-team/rocm-hipamd/-/raw/d6d20142c37e1dff820950b16ff8f0523241d935/debian/patches/0025-improve-rocclr-isa-compatibility-check.patch"; + hash = "sha256-8eowuRiOAdd9ucKv4Eg9FPU7c6367H3eP3fRAGfXc6Y="; }) ]; From 45b3242ab43885004198f9ae0bd79db4b7d97abe Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Wed, 3 Apr 2024 01:28:19 -0400 Subject: [PATCH 12/15] rocmPackages: add isa conversion patch in rocprim Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/rocprim/default.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkgs/development/rocm-modules/6/rocprim/default.nix b/pkgs/development/rocm-modules/6/rocprim/default.nix index e9996cae61e78..3e8525655141e 100644 --- a/pkgs/development/rocm-modules/6/rocprim/default.nix +++ b/pkgs/development/rocm-modules/6/rocprim/default.nix @@ -1,4 +1,5 @@ { lib +, fetchpatch , stdenv , fetchFromGitHub , rocmUpdateScript @@ -31,6 +32,14 @@ stdenv.mkDerivation (finalAttrs: { hash = "sha256-nWvq26qRPZ6Au1rc5cR74TKArcdUFg7O9djFi8SvMeM="; }; + patches = [ + (fetchpatch { + name = "arch-conversion-marco.patch"; + url = "https://salsa.debian.org/rocm-team/rocprim/-/raw/70c8aaee3cf545d92685f4ed9bf8f41e3d4d570c/debian/patches/arch-conversion-macro.patch"; + hash = "sha256-oXdmbCArOB5bKE8ozDFrSh4opbO+c4VI6PNhljeUSms="; + }) + ]; + nativeBuildInputs = [ cmake rocm-cmake From 7bbde1756d70eacb2f3ba5ec8e8a34de28e801b2 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Wed, 3 Apr 2024 02:13:22 -0400 Subject: [PATCH 13/15] rocmPackages: no need to pass rocblas to itself Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/rocm-modules/6/default.nix b/pkgs/development/rocm-modules/6/default.nix index 567b5bee4f728..e74df33a7bfbf 100644 --- a/pkgs/development/rocm-modules/6/default.nix +++ b/pkgs/development/rocm-modules/6/default.nix @@ -194,7 +194,7 @@ in rec { }; rocblas = callPackage ./rocblas { - inherit rocblas rocmUpdateScript rocm-cmake clr tensile; + inherit rocmUpdateScript rocm-cmake clr tensile; inherit (llvm) openmp; stdenv = llvm.rocmClangStdenv; }; From 32b9f37d57826364db0bc579ee75fa82af1fd623 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Wed, 24 Apr 2024 22:22:11 -0400 Subject: [PATCH 14/15] rocmPackages.rocblas: do not build for gfx1012, gfx940, and gfx 941 GPU targets Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/rocblas/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/rocm-modules/6/rocblas/default.nix b/pkgs/development/rocm-modules/6/rocblas/default.nix index 9dfa8636f1201..6bfb765322631 100644 --- a/pkgs/development/rocm-modules/6/rocblas/default.nix +++ b/pkgs/development/rocm-modules/6/rocblas/default.nix @@ -29,7 +29,7 @@ , tensileSepArch ? false , tensileLazyLib ? false , tensileLibFormat ? "msgpack" -, gpuTargets ? [ "all" ] +, gpuTargets ? [ "gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx942;gfx1010;gfx1030;gfx1100;gfx1101;gfx1102" ] }: stdenv.mkDerivation (finalAttrs: { From c3848d59c9506ce6e54e6c1d52670a6191e427ea Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Thu, 25 Apr 2024 11:45:45 -0400 Subject: [PATCH 15/15] rocmPackages.rocblas: add comments about why some GPU archs are not built Signed-off-by: Gavin Zhao --- pkgs/development/rocm-modules/6/rocblas/default.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkgs/development/rocm-modules/6/rocblas/default.nix b/pkgs/development/rocm-modules/6/rocblas/default.nix index 6bfb765322631..f93cceddd68af 100644 --- a/pkgs/development/rocm-modules/6/rocblas/default.nix +++ b/pkgs/development/rocm-modules/6/rocblas/default.nix @@ -29,6 +29,14 @@ , tensileSepArch ? false , tensileLazyLib ? false , tensileLibFormat ? "msgpack" +# `gfx940`, `gfx941` are not present in this list because they are early +# engineering samples, and all final MI300 hardware are `gfx942`: +# https://github.com/NixOS/nixpkgs/pull/298388#issuecomment-2032791130 +# +# `gfx1012` is not present in this list because the ISA compatibility patches +# would force all `gfx101*` GPUs to run as `gfx1010`, so `gfx101*` GPUs will +# always try to use `gfx1010` code objects, hence building for `gfx1012` is +# useless: https://github.com/NixOS/nixpkgs/pull/298388#issuecomment-2076327152 , gpuTargets ? [ "gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx942;gfx1010;gfx1030;gfx1100;gfx1101;gfx1102" ] }: