From 6e69f004639de5a86af51aa6d9b89ef5c289ad75 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Mon, 23 Oct 2023 18:07:03 -0500
Subject: [PATCH 01/19] python3Packages.openai-triton: don't download ptxas for
 non-cuda build

Use openai-triton-cuda for derivations that previously used just openai
---
 .../0000-dont-download-ptxas.patch            | 39 +++++++++++++++++++
 .../python-modules/openai-triton/default.nix  | 20 ++++++----
 pkgs/top-level/python-packages.nix            | 23 +++++++++--
 3 files changed, 71 insertions(+), 11 deletions(-)
 create mode 100644 pkgs/development/python-modules/openai-triton/0000-dont-download-ptxas.patch

diff --git a/pkgs/development/python-modules/openai-triton/0000-dont-download-ptxas.patch b/pkgs/development/python-modules/openai-triton/0000-dont-download-ptxas.patch
new file mode 100644
index 0000000000000..5038a5ba52e05
--- /dev/null
+++ b/pkgs/development/python-modules/openai-triton/0000-dont-download-ptxas.patch
@@ -0,0 +1,39 @@
+diff --git a/python/setup.py b/python/setup.py
+index 2ac3accd2..f26161c72 100644
+--- a/python/setup.py
++++ b/python/setup.py
+@@ -101,25 +101,6 @@ def get_thirdparty_packages(triton_cache_path):
+ # ---- package data ---
+ 
+ 
+-def download_and_copy_ptxas():
+-    base_dir = os.path.dirname(__file__)
+-    src_path = "bin/ptxas"
+-    url = "https://conda.anaconda.org/nvidia/label/cuda-12.0.0/linux-64/cuda-nvcc-12.0.76-0.tar.bz2"
+-    dst_prefix = os.path.join(base_dir, "triton")
+-    dst_suffix = os.path.join("third_party", "cuda", src_path)
+-    dst_path = os.path.join(dst_prefix, dst_suffix)
+-    if not os.path.exists(dst_path):
+-        print(f'downloading and extracting {url} ...')
+-        ftpstream = urllib.request.urlopen(url)
+-        file = tarfile.open(fileobj=ftpstream, mode="r|*")
+-        with tempfile.TemporaryDirectory() as temp_dir:
+-            file.extractall(path=temp_dir)
+-            src_path = os.path.join(temp_dir, src_path)
+-            os.makedirs(os.path.split(dst_path)[0], exist_ok=True)
+-            shutil.copy(src_path, dst_path)
+-    return dst_suffix
+-
+-
+ # ---- cmake extension ----
+ 
+ 
+@@ -200,8 +181,6 @@ class CMakeBuild(build_ext):
+         subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp)
+ 
+ 
+-download_and_copy_ptxas()
+-
+ setup(
+     name="triton",
+     version="2.0.0",
diff --git a/pkgs/development/python-modules/openai-triton/default.nix b/pkgs/development/python-modules/openai-triton/default.nix
index 68dfc24aafdaa..89545f032657d 100644
--- a/pkgs/development/python-modules/openai-triton/default.nix
+++ b/pkgs/development/python-modules/openai-triton/default.nix
@@ -1,5 +1,6 @@
 { lib
 , callPackage
+, config
 , buildPythonPackage
 , fetchFromGitHub
 , addOpenGLRunpath
@@ -18,6 +19,7 @@
 , torchWithRocm
 , python
 , cudaPackages
+, cudaSupport ? config.cudaSupport
 }:
 
 let
@@ -57,6 +59,8 @@ buildPythonPackage rec {
     #   url = "https://github.com/openai/triton/commit/fc7c0b0e437a191e421faa61494b2ff4870850f1.patch";
     #   hash = "sha256-f0shIqHJkVvuil2Yku7vuqWFn7VCRKFSFjYRlwx25ig=";
     # })
+  ] ++ lib.optionals (!cudaSupport) [
+    ./0000-dont-download-ptxas.patch
   ];
 
   nativeBuildInputs = [
@@ -102,21 +106,18 @@ buildPythonPackage rec {
     substituteInPlace bin/CMakeLists.txt \
       --replace "add_subdirectory(FileCheck)" ""
 
-    # Use our linker flags
-    substituteInPlace python/triton/compiler.py \
-      --replace '${oldStr}' '${newStr}'
-
     # Don't fetch googletest
     substituteInPlace unittest/CMakeLists.txt \
       --replace "include (\''${CMAKE_CURRENT_SOURCE_DIR}/googletest.cmake)" ""\
       --replace "include(GoogleTest)" "find_package(GTest REQUIRED)"
+  '' + lib.optionalString cudaSupport ''
+    # Use our linker flags
+    substituteInPlace python/triton/compiler.py \
+      --replace '${oldStr}' '${newStr}'
   '';
 
   # Avoid GLIBCXX mismatch with other cuda-enabled python packages
   preConfigure = ''
-    export CC=${cudaPackages.backendStdenv.cc}/bin/cc;
-    export CXX=${cudaPackages.backendStdenv.cc}/bin/c++;
-
     # Upstream's setup.py tries to write cache somewhere in ~/
     export HOME=$(mktemp -d)
 
@@ -127,6 +128,9 @@ buildPythonPackage rec {
 
     # The rest (including buildPhase) is relative to ./python/
     cd python
+  '' + lib.optionalString cudaSupport ''
+    export CC=${cudaPackages.backendStdenv.cc}/bin/cc;
+    export CXX=${cudaPackages.backendStdenv.cc}/bin/c++;
 
     # Work around download_and_copy_ptxas()
     mkdir -p $PWD/triton/third_party/cuda/bin
@@ -137,7 +141,7 @@ buildPythonPackage rec {
   dontUseCmakeConfigure = true;
 
   # Setuptools (?) strips runpath and +x flags. Let's just restore the symlink
-  postFixup = ''
+  postFixup = lib.optionalString cudaSupport ''
     rm -f $out/${python.sitePackages}/triton/third_party/cuda/bin/ptxas
     ln -s ${ptxas} $out/${python.sitePackages}/triton/third_party/cuda/bin/ptxas
   '';
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 6368717af8203..31e66b5fa91b4 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -4160,7 +4160,9 @@ self: super: with self; {
 
   oelint-parser = callPackage ../development/python-modules/oelint-parser { };
 
-  openllm = callPackage ../development/python-modules/openllm { };
+  openllm = callPackage ../development/python-modules/openllm {
+    openai-triton = self.openai-triton-cuda;
+  };
 
   openllm-client = callPackage ../development/python-modules/openllm-client { };
 
@@ -8427,12 +8429,23 @@ self: super: with self; {
 
   open-meteo = callPackage ../development/python-modules/open-meteo { };
 
-  openai-triton = callPackage ../development/python-modules/openai-triton { cudaPackages = pkgs.cudaPackages_12_0; };
+  openai-triton = callPackage ../development/python-modules/openai-triton {
+    cudaPackages = pkgs.cudaPackages_12_0;
+  };
+
+  openai-triton-cuda = self.openai-triton.override {
+    cudaSupport = true;
+  };
+
+  openai-triton-no-cuda = self.openai-triton.override {
+    cudaSupport = false;
+  };
 
   openai-triton-bin = callPackage ../development/python-modules/openai-triton/bin.nix { };
 
   openai-whisper = callPackage ../development/python-modules/openai-whisper {
     inherit (pkgs.config) cudaSupport;
+    openai-triton = self.openai-triton-cuda;
   };
 
   openant = callPackage ../development/python-modules/openant { };
@@ -14064,6 +14077,7 @@ self: super: with self; {
 
   torchWithCuda = self.torch.override {
     magma = pkgs.magma-cuda-static;
+    openai-triton = self.openai-triton-cuda;
     cudaSupport = true;
     rocmSupport = false;
   };
@@ -14074,6 +14088,7 @@ self: super: with self; {
 
   torchWithRocm = self.torch.override {
     magma = pkgs.magma-hip;
+    openai-triton = self.openai-triton-no-cuda;
     rocmSupport = true;
     cudaSupport = false;
   };
@@ -15772,7 +15787,9 @@ self: super: with self; {
     inherit (pkgs) graphviz;
   };
 
-  xformers = callPackage ../development/python-modules/xformers { };
+  xformers = callPackage ../development/python-modules/xformers {
+    openai-triton = self.openai-triton-cuda;
+  };
 
   xgboost = callPackage ../development/python-modules/xgboost {
     inherit (pkgs) xgboost;

From ac0b05fe7fd1ad493b47a217a191459119014332 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Mon, 23 Oct 2023 18:09:33 -0500
Subject: [PATCH 02/19] python3Packages.torch: include openai-triton in
 propagatedBuildInputs by default

---
 pkgs/development/python-modules/torch/default.nix | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/pkgs/development/python-modules/torch/default.nix b/pkgs/development/python-modules/torch/default.nix
index 52ab9ee5b25df..b9284b14eb9ef 100644
--- a/pkgs/development/python-modules/torch/default.nix
+++ b/pkgs/development/python-modules/torch/default.nix
@@ -342,17 +342,12 @@ in buildPythonPackage rec {
 
     # the following are required for tensorboard support
     pillow six future tensorboard protobuf
+
+    # ROCm build and `torch.compile` requires openai-triton
+    openai-triton
   ]
   ++ lib.optionals MPISupport [ mpi ]
-  ++ lib.optionals rocmSupport [ rocmtoolkit_joined ]
-  # rocm build requires openai-triton;
-  # openai-triton currently requires cuda_nvcc,
-  # so not including it in the cpu-only build;
-  # torch.compile relies on openai-triton,
-  # so we include it for the cuda build as well
-  ++ lib.optionals (rocmSupport || cudaSupport) [
-    openai-triton
-  ];
+  ++ lib.optionals rocmSupport [ rocmtoolkit_joined ];
 
   # Tests take a long time and may be flaky, so just sanity-check imports
   doCheck = false;

From b5dce72180fe507a3050e6e759b3e8bbdd74c81c Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Mon, 23 Oct 2023 18:39:33 -0500
Subject: [PATCH 03/19] openai-triton-llvm: init at 14.0.6-f28c006a5895

Moved from python3Packages.openai-triton due to rebuilds per python version
---
 .../llvm.nix => by-name/op/openai-triton-llvm/package.nix}     | 2 +-
 pkgs/development/python-modules/openai-triton/default.nix      | 3 +--
 pkgs/top-level/python-packages.nix                             | 1 +
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename pkgs/{development/python-modules/openai-triton/llvm.nix => by-name/op/openai-triton-llvm/package.nix} (99%)

diff --git a/pkgs/development/python-modules/openai-triton/llvm.nix b/pkgs/by-name/op/openai-triton-llvm/package.nix
similarity index 99%
rename from pkgs/development/python-modules/openai-triton/llvm.nix
rename to pkgs/by-name/op/openai-triton-llvm/package.nix
index 70ea69a9b15fe..2fb56d0a63522 100644
--- a/pkgs/development/python-modules/openai-triton/llvm.nix
+++ b/pkgs/by-name/op/openai-triton-llvm/package.nix
@@ -22,7 +22,7 @@
 }:
 
 stdenv.mkDerivation (finalAttrs: {
-  pname = "triton-llvm";
+  pname = "openai-triton-llvm";
   version = "14.0.6-f28c006a5895";
 
   outputs = [
diff --git a/pkgs/development/python-modules/openai-triton/default.nix b/pkgs/development/python-modules/openai-triton/default.nix
index 89545f032657d..e1ac9cb4cef6e 100644
--- a/pkgs/development/python-modules/openai-triton/default.nix
+++ b/pkgs/development/python-modules/openai-triton/default.nix
@@ -1,5 +1,4 @@
 { lib
-, callPackage
 , config
 , buildPythonPackage
 , fetchFromGitHub
@@ -15,6 +14,7 @@
 , ncurses
 , libxml2
 , lit
+, llvm
 , filelock
 , torchWithRocm
 , python
@@ -35,7 +35,6 @@ let
   # be executed on the GPU.
   # Cf. https://nixos.org/manual/nixpkgs/unstable/#sec-cross-infra
   ptxas = "${pkgsTargetTarget.cudaPackages.cuda_nvcc}/bin/ptxas"; # Make sure cudaPackages is the right version each update (See python/setup.py)
-  llvm = callPackage ./llvm.nix { }; # Use a custom llvm, see llvm.nix for details
 in
 buildPythonPackage rec {
   pname = "triton";
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 31e66b5fa91b4..9d4be43e17d79 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -8430,6 +8430,7 @@ self: super: with self; {
   open-meteo = callPackage ../development/python-modules/open-meteo { };
 
   openai-triton = callPackage ../development/python-modules/openai-triton {
+    llvm = pkgs.openai-triton-llvm;
     cudaPackages = pkgs.cudaPackages_12_0;
   };
 

From b280deae9b19a14e404d2d69cd9b373f870a76ec Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 12:57:53 -0500
Subject: [PATCH 04/19] rapidjson-unstable: rewrite and fix tests

Some valgrind tests do not pass, see https://github.com/Tencent/rapidjson/issues/2214
---
 .../0000-unstable-use-nixpkgs-gtest.patch     | 35 ++++++++++
 ...-unstable-valgrind-suppress-failures.patch | 36 ++++++++++
 .../libraries/rapidjson/unstable.nix          | 66 ++++++++++++++-----
 3 files changed, 120 insertions(+), 17 deletions(-)
 create mode 100644 pkgs/development/libraries/rapidjson/0000-unstable-use-nixpkgs-gtest.patch
 create mode 100644 pkgs/development/libraries/rapidjson/0001-unstable-valgrind-suppress-failures.patch

diff --git a/pkgs/development/libraries/rapidjson/0000-unstable-use-nixpkgs-gtest.patch b/pkgs/development/libraries/rapidjson/0000-unstable-use-nixpkgs-gtest.patch
new file mode 100644
index 0000000000000..df334cc912642
--- /dev/null
+++ b/pkgs/development/libraries/rapidjson/0000-unstable-use-nixpkgs-gtest.patch
@@ -0,0 +1,35 @@
+diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
+index 11c1b04c..762eaa75 100644
+--- a/test/CMakeLists.txt
++++ b/test/CMakeLists.txt
+@@ -1,20 +1,14 @@
+-find_package(GTestSrc)
++enable_testing()
+ 
+-IF(GTESTSRC_FOUND)
+-    enable_testing()
++if (WIN32 AND (NOT CYGWIN) AND (NOT MINGW))
++    set(gtest_disable_pthreads ON)
++    set(gtest_force_shared_crt ON)
++endif()
+ 
+-    if (WIN32 AND (NOT CYGWIN) AND (NOT MINGW))
+-        set(gtest_disable_pthreads ON)
+-        set(gtest_force_shared_crt ON)
+-    endif()
++include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
+ 
+-    add_subdirectory(${GTEST_SOURCE_DIR} ${CMAKE_BINARY_DIR}/googletest)
+-    include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
++set(TEST_LIBRARIES gtest gtest_main)
+ 
+-    set(TEST_LIBRARIES gtest gtest_main)
+-
+-    add_custom_target(tests ALL)
+-    add_subdirectory(perftest)
+-    add_subdirectory(unittest)
+-
+-ENDIF(GTESTSRC_FOUND)
++add_custom_target(tests ALL)
++add_subdirectory(perftest)
++add_subdirectory(unittest)
diff --git a/pkgs/development/libraries/rapidjson/0001-unstable-valgrind-suppress-failures.patch b/pkgs/development/libraries/rapidjson/0001-unstable-valgrind-suppress-failures.patch
new file mode 100644
index 0000000000000..dd06ba6a69711
--- /dev/null
+++ b/pkgs/development/libraries/rapidjson/0001-unstable-valgrind-suppress-failures.patch
@@ -0,0 +1,36 @@
+diff --git a/test/valgrind.supp b/test/valgrind.supp
+index c9d3d226..63af7bf9 100644
+--- a/test/valgrind.supp
++++ b/test/valgrind.supp
+@@ -24,3 +24,31 @@
+    fun:*Uri*Parse_UTF16_Std*
+ }
+ 
++{
++   Suppress memcpy_chk valgrind report 5
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   fun:*dtoa_normal*
++}
++
++{
++   Suppress memcpy_chk valgrind report 6
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   fun:*dtoa_maxDecimalPlaces*
++}
++
++{
++   Suppress memcpy_chk valgrind report 7
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   ...
++   fun:*Reader*ParseDoubleHandler*
++}
++
++{
++   Suppress memcpy_chk valgrind report 8
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   fun:*Reader*ParseNumber_NormalPrecisionError*
++}
diff --git a/pkgs/development/libraries/rapidjson/unstable.nix b/pkgs/development/libraries/rapidjson/unstable.nix
index 069f1a48e0ea7..0f4c3d40403e1 100644
--- a/pkgs/development/libraries/rapidjson/unstable.nix
+++ b/pkgs/development/libraries/rapidjson/unstable.nix
@@ -1,17 +1,30 @@
 { lib
 , stdenv
 , fetchFromGitHub
-, fetchpatch
-, pkg-config
 , cmake
+, doxygen
+, graphviz
 , gtest
 , valgrind
+# One of "11" or "17"; default in source is CXX 11
+, cxxStandard ? "11"
+, buildDocs ? true
+, buildTests ? !stdenv.hostPlatform.isStatic && !stdenv.isDarwin
+, buildExamples ? true
 }:
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: {
   pname = "rapidjson";
   version = "unstable-2023-09-28";
 
+  outputs = [
+    "out"
+  ] ++ lib.optionals buildDocs [
+    "doc"
+  ] ++ lib.optionals buildExamples [
+    "example"
+  ];
+
   src = fetchFromGitHub {
     owner = "Tencent";
     repo = "rapidjson";
@@ -19,26 +32,44 @@ stdenv.mkDerivation rec {
     hash = "sha256-rl7iy14jn1K2I5U2DrcZnoTQVEGEDKlxmdaOCF/3hfY=";
   };
 
+  patches = lib.optionals buildTests [
+    ./0000-unstable-use-nixpkgs-gtest.patch
+    # https://github.com/Tencent/rapidjson/issues/2214
+    ./0001-unstable-valgrind-suppress-failures.patch
+  ];
+
   nativeBuildInputs = [
-    pkg-config
     cmake
+  ] ++ lib.optionals buildDocs [
+    doxygen
+    graphviz
   ];
 
-  patches = [
-    (fetchpatch {
-      name = "do-not-include-gtest-src-dir.patch";
-      url = "https://git.alpinelinux.org/aports/plain/community/rapidjson/do-not-include-gtest-src-dir.patch?id=9e5eefc7a5fcf5938a8dc8a3be8c75e9e6809909";
-      hash = "sha256-BjSZEwfCXA/9V+kxQ/2JPWbc26jQn35CfN8+8NW24s4=";
-    })
+  cmakeFlags = [
+    (lib.cmakeBool "RAPIDJSON_BUILD_DOC" buildDocs)
+    (lib.cmakeBool "RAPIDJSON_BUILD_TESTS" buildTests)
+    (lib.cmakeBool "RAPIDJSON_BUILD_EXAMPLES" buildExamples)
+    (lib.cmakeBool "RAPIDJSON_BUILD_CXX11" (cxxStandard == "11"))
+    (lib.cmakeBool "RAPIDJSON_BUILD_CXX17" (cxxStandard == "17"))
+  ] ++ lib.optionals buildTests [
+    (lib.cmakeFeature "GTEST_INCLUDE_DIR" "${lib.getDev gtest}")
+  ];
+
+  doCheck = buildTests;
+
+  nativeCheckInputs = [
+    gtest
+    valgrind
   ];
 
-  # for tests, adding gtest to checkInputs does not work
-  # https://github.com/NixOS/nixpkgs/pull/212200
-  buildInputs = [ gtest ];
-  cmakeFlags = [ "-DGTEST_SOURCE_DIR=${gtest.dev}/include" ];
+  postInstall = lib.optionalString buildExamples ''
+    mkdir -p $example/bin
 
-  nativeCheckInputs = [ valgrind ];
-  doCheck = !stdenv.hostPlatform.isStatic && !stdenv.isDarwin;
+    find bin -type f -executable \
+      -not -name "perftest" \
+      -not -name "unittest" \
+      -exec cp -a {} $example/bin \;
+  '';
 
   meta = with lib; {
     description = "Fast JSON parser/generator for C++ with both SAX/DOM style API";
@@ -46,5 +77,6 @@ stdenv.mkDerivation rec {
     license = licenses.mit;
     platforms = platforms.unix;
     maintainers = with maintainers; [ Madouura ];
+    broken = (cxxStandard != "11" && cxxStandard != "17");
   };
-}
+})

From a92172bf6538eb4a9c5ed36295ef839245a888d7 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 13:02:59 -0500
Subject: [PATCH 05/19] python3Packages.pytorch-pfn-extras: fixup tests

---
 .../development/python-modules/pytorch-pfn-extras/default.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkgs/development/python-modules/pytorch-pfn-extras/default.nix b/pkgs/development/python-modules/pytorch-pfn-extras/default.nix
index 138dc5f502fa4..04146f9bac740 100644
--- a/pkgs/development/python-modules/pytorch-pfn-extras/default.nix
+++ b/pkgs/development/python-modules/pytorch-pfn-extras/default.nix
@@ -41,6 +41,7 @@ buildPythonPackage rec {
     "tests/pytorch_pfn_extras_tests/onnx_tests/test_export.py"
     "tests/pytorch_pfn_extras_tests/onnx_tests/test_torchvision.py"
     "tests/pytorch_pfn_extras_tests/onnx_tests/utils.py"
+    "tests/pytorch_pfn_extras_tests/onnx_tests/test_lax.py"
 
     # RuntimeError: No Op registered for Gradient with domain_version of 9
     "tests/pytorch_pfn_extras_tests/onnx_tests/test_grad.py"
@@ -53,8 +54,7 @@ buildPythonPackage rec {
     "tests/pytorch_pfn_extras_tests/nn_tests/modules_tests/test_lazy.py"
     "tests/pytorch_pfn_extras_tests/profiler_tests/test_record.py"
     "tests/pytorch_pfn_extras_tests/runtime_tests/test_to.py"
-    "tests/pytorch_pfn_extras_tests/test_handler.py"
-    "tests/pytorch_pfn_extras_tests/test_logic.py"
+    "tests/pytorch_pfn_extras_tests/handler_tests/test_handler.py"
     "tests/pytorch_pfn_extras_tests/test_reporter.py"
     "tests/pytorch_pfn_extras_tests/training_tests/test_trainer.py"
     "tests/pytorch_pfn_extras_tests/utils_tests/test_checkpoint.py"

From 071f6a5fb28d9cf34dbd688ee2de90f2ec5f59f3 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 13:08:28 -0500
Subject: [PATCH 06/19] python3Packages.pytorch-pfn-extras: fixup tests for
 python 3.11

Revert this commit when https://github.com/NixOS/nixpkgs/pull/259068 is merged
---
 .../development/python-modules/pytorch-pfn-extras/default.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkgs/development/python-modules/pytorch-pfn-extras/default.nix b/pkgs/development/python-modules/pytorch-pfn-extras/default.nix
index 04146f9bac740..56473cc95ae26 100644
--- a/pkgs/development/python-modules/pytorch-pfn-extras/default.nix
+++ b/pkgs/development/python-modules/pytorch-pfn-extras/default.nix
@@ -8,6 +8,7 @@
 , torch
 , torchvision
 , typing-extensions
+, pythonAtLeast
 }:
 
 buildPythonPackage rec {
@@ -60,6 +61,9 @@ buildPythonPackage rec {
     "tests/pytorch_pfn_extras_tests/utils_tests/test_checkpoint.py"
     "tests/pytorch_pfn_extras_tests/utils_tests/test_comparer.py"
     "tests/pytorch_pfn_extras_tests/utils_tests/test_new_comparer.py"
+  ] ++ lib.optionals (pythonAtLeast "3.11") [
+    # Remove this when https://github.com/NixOS/nixpkgs/pull/259068 is merged
+    "tests/pytorch_pfn_extras_tests/dynamo_tests/test_compile.py"
   ];
 
   meta = with lib; {

From b45ba74486358600ac4cc3e3454356abc2362e24 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 13:20:36 -0500
Subject: [PATCH 07/19] python3Packages.arviz: fixup tests

---
 pkgs/development/python-modules/arviz/default.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkgs/development/python-modules/arviz/default.nix b/pkgs/development/python-modules/arviz/default.nix
index b37e7d9a8a189..0a186781b4a22 100644
--- a/pkgs/development/python-modules/arviz/default.nix
+++ b/pkgs/development/python-modules/arviz/default.nix
@@ -28,6 +28,7 @@
 , pyro-ppl
   #, pystan (not packaged)
 , numpyro
+, bokeh
 }:
 
 buildPythonPackage rec {
@@ -72,6 +73,7 @@ buildPythonPackage rec {
     pytestCheckHook
     torchvision
     zarr
+    bokeh
   ];
 
   preCheck = ''
@@ -97,6 +99,8 @@ buildPythonPackage rec {
     # An issue has been opened upstream: https://github.com/arviz-devs/arviz/issues/2282
     "test_plot_ppc_discrete"
     "test_plot_ppc_discrete_save_animation"
+    # Assertion error
+    "test_data_zarr"
   ];
 
   pythonImportsCheck = [

From f7f968c005cba095c13dd00679822352d15f9360 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 13:31:04 -0500
Subject: [PATCH 08/19] python3Packages.cvxpy: 1.3.2 -> 1.4.1

---
 pkgs/development/python-modules/cvxpy/default.nix | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pkgs/development/python-modules/cvxpy/default.nix b/pkgs/development/python-modules/cvxpy/default.nix
index 0fb8be981e101..3d34b6edda8a2 100644
--- a/pkgs/development/python-modules/cvxpy/default.nix
+++ b/pkgs/development/python-modules/cvxpy/default.nix
@@ -12,19 +12,20 @@
 , scs
 , setuptools
 , wheel
+, pybind11
 , useOpenmp ? (!stdenv.isDarwin)
 }:
 
 buildPythonPackage rec {
   pname = "cvxpy";
-  version = "1.3.2";
+  version = "1.4.1";
   format = "pyproject";
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    hash = "sha256-C2heUEDxmfPXA/MPXSLR+GVZdiNFUVPR3ddwJFrvCXU=";
+    hash = "sha256-ep7zTjxX/4yETYbwo4NPtVda8ZIzlHY53guld8YSLj4=";
   };
 
   # we need to patch out numpy version caps from upstream
@@ -35,6 +36,7 @@ buildPythonPackage rec {
   nativeBuildInputs = [
     setuptools
     wheel
+    pybind11
   ];
 
   propagatedBuildInputs = [
@@ -44,7 +46,6 @@ buildPythonPackage rec {
     osqp
     scipy
     scs
-    setuptools
   ];
 
   nativeCheckInputs = [

From fec04ec93bf2d4dd83e55dd45936dc7e9321867a Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 13:55:43 -0500
Subject: [PATCH 09/19] python3Packages.torchinfo: disable failing test

---
 pkgs/development/python-modules/torchinfo/default.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkgs/development/python-modules/torchinfo/default.nix b/pkgs/development/python-modules/torchinfo/default.nix
index 6cc7bcf3e27c9..59b7e99c9dfca 100644
--- a/pkgs/development/python-modules/torchinfo/default.nix
+++ b/pkgs/development/python-modules/torchinfo/default.nix
@@ -53,6 +53,8 @@ buildPythonPackage rec {
     "test_flan_t5_small"
     # AssertionError in output
     "test_google"
+    # "addmm_impl_cpu_" not implemented for 'Half'
+    "test_input_size_half_precision"
   ];
 
   disabledTestPaths = [

From 1f25043964b8d61299c2b28c09aaac53890bf907 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 14:05:51 -0500
Subject: [PATCH 10/19] python3Packages.ax: disable failing tests

---
 pkgs/development/python-modules/ax/default.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkgs/development/python-modules/ax/default.nix b/pkgs/development/python-modules/ax/default.nix
index e9ca9dba6ca73..1a4da01df6d43 100644
--- a/pkgs/development/python-modules/ax/default.nix
+++ b/pkgs/development/python-modules/ax/default.nix
@@ -69,6 +69,10 @@ buildPythonPackage rec {
   disabledTests = [
     # exact comparison of floating points
     "test_optimize_l0_homotopy"
+    # AssertionError: 5 != 2
+    "test_get_standard_plots_moo"
+    # AssertionError: Expected 'warning' to be called once. Called 3 times
+    "test_validate_kwarg_typing"
   ];
   pythonImportsCheck = [ "ax" ];
 

From 0775491b8b7e25ccb03c69a4a9c8888e22c39c47 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 14:35:08 -0500
Subject: [PATCH 11/19] python3Packages.stytra: fix tests

---
 .../stytra/0000-workaround-pyqtgraph.patch    | 26 +++++++++++++++++++
 .../python-modules/stytra/default.nix         | 16 +++++++-----
 2 files changed, 36 insertions(+), 6 deletions(-)
 create mode 100644 pkgs/development/python-modules/stytra/0000-workaround-pyqtgraph.patch

diff --git a/pkgs/development/python-modules/stytra/0000-workaround-pyqtgraph.patch b/pkgs/development/python-modules/stytra/0000-workaround-pyqtgraph.patch
new file mode 100644
index 0000000000000..ad0ecbb113016
--- /dev/null
+++ b/pkgs/development/python-modules/stytra/0000-workaround-pyqtgraph.patch
@@ -0,0 +1,26 @@
+diff --git a/stytra/gui/fishplots.py b/stytra/gui/fishplots.py
+index 49ef1fe..fd1cc50 100644
+--- a/stytra/gui/fishplots.py
++++ b/stytra/gui/fishplots.py
+@@ -13,7 +13,7 @@ from lightparam.gui import ParameterGui
+ from scipy.ndimage.filters import gaussian_filter1d
+ 
+ 
+-class StreamingPositionPlot(pg.GraphicsWindow):
++class StreamingPosition(pg.GraphicsView):
+     """Plot that displays the virtual position of the fish"""
+ 
+     def __init__(self, *args, data_accumulator, n_points=500, **kwargs):
+diff --git a/stytra/utilities.py b/stytra/utilities.py
+index f79c4db..feaa7ef 100644
+--- a/stytra/utilities.py
++++ b/stytra/utilities.py
+@@ -239,7 +239,7 @@ def recursive_update(d, u):
+     :return:
+     """
+     for k, v in u.items():
+-        if isinstance(v, collections.Mapping):
++        if isinstance(v, collections.ChainMap):
+             d[k] = recursive_update(d.get(k, {}), v)
+         else:
+             d[k] = v
diff --git a/pkgs/development/python-modules/stytra/default.nix b/pkgs/development/python-modules/stytra/default.nix
index 01907cb15b03d..821aaa41437c8 100644
--- a/pkgs/development/python-modules/stytra/default.nix
+++ b/pkgs/development/python-modules/stytra/default.nix
@@ -33,8 +33,7 @@
 buildPythonPackage rec {
   pname = "stytra";
   version = "0.8.34";
-  format = "setuptools";
-
+  pyproject = true;
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
@@ -42,10 +41,10 @@ buildPythonPackage rec {
     sha256 = "aab9d07575ef599a9c0ae505656e3c03ec753462df3c15742f1f768f2b578f0a";
   };
 
-  # crashes python
-  preCheck = ''
-    rm stytra/tests/test_z_experiments.py
-  '';
+  patches = [
+    # https://github.com/portugueslab/stytra/issues/87
+    ./0000-workaround-pyqtgraph.patch
+  ];
 
   propagatedBuildInputs = [
     opencv4
@@ -79,6 +78,11 @@ buildPythonPackage rec {
     pyserial
   ];
 
+  disabledTestPaths = [
+    # Crashes python
+    "stytra/tests/test_z_experiments.py"
+  ];
+
   meta = with lib; {
     description = "A modular package to control stimulation and track behaviour";
     homepage = "https://github.com/portugueslab/stytra";

From 956abdcaeefa5e8f8306004ec8f3c8b347f9b622 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 14:57:00 -0500
Subject: [PATCH 12/19] python3Packages.mayavi: disable for python 3.11

---
 pkgs/development/python-modules/mayavi/default.nix | 4 ++--
 pkgs/top-level/python-packages.nix                 | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/pkgs/development/python-modules/mayavi/default.nix b/pkgs/development/python-modules/mayavi/default.nix
index bdc0a74007d81..dc786ac6d7d68 100644
--- a/pkgs/development/python-modules/mayavi/default.nix
+++ b/pkgs/development/python-modules/mayavi/default.nix
@@ -9,6 +9,7 @@
 , pygments
 , pyqt5
 , pythonOlder
+, pythonAtLeast
 , traitsui
 , vtk
 , wrapQtAppsHook
@@ -18,8 +19,7 @@ buildPythonPackage rec {
   pname = "mayavi";
   version = "4.8.1";
   format = "setuptools";
-
-  disabled = pythonOlder "3.8";
+  disabled = pythonOlder "3.8" || pythonAtLeast "3.11";
 
   src = fetchPypi {
     inherit pname version;
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 9d4be43e17d79..d2cb83041a591 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -6662,8 +6662,7 @@ self: super: with self; {
   maya = callPackage ../development/python-modules/maya { };
 
   mayavi = pkgs.libsForQt5.callPackage ../development/python-modules/mayavi {
-    inherit buildPythonPackage pythonOlder;
-    inherit (self) pyface pygments numpy packaging vtk traitsui envisage apptools pyqt5;
+    inherit (self) buildPythonPackage pythonOlder pythonAtLeast pyface pygments numpy packaging vtk traitsui envisage apptools pyqt5;
   };
 
   mbddns = callPackage ../development/python-modules/mbddns { };

From 6d7ae4f4269bac3032a14da3cc9a2fec17e29143 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Tue, 24 Oct 2023 16:42:25 -0500
Subject: [PATCH 13/19] python3Packages.torch: propagate pybind11

---
 pkgs/development/python-modules/torch/default.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pkgs/development/python-modules/torch/default.nix b/pkgs/development/python-modules/torch/default.nix
index b9284b14eb9ef..a1b9c82dba574 100644
--- a/pkgs/development/python-modules/torch/default.nix
+++ b/pkgs/development/python-modules/torch/default.nix
@@ -291,7 +291,7 @@ in buildPythonPackage rec {
   ])
   ++ lib.optionals rocmSupport [ rocmtoolkit_joined ];
 
-  buildInputs = [ blas blas.provider pybind11 ]
+  buildInputs = [ blas blas.provider ]
     ++ lib.optionals stdenv.isLinux [ linuxHeaders_5_19 ] # TMP: avoid "flexible array member" errors for now
     ++ lib.optionals cudaSupport (with cudaPackages; [
       cuda_cccl.dev # <thrust/*>
@@ -345,6 +345,9 @@ in buildPythonPackage rec {
 
     # ROCm build and `torch.compile` requires openai-triton
     openai-triton
+
+    # torch/csrc requires `pybind11` at runtime
+    pybind11
   ]
   ++ lib.optionals MPISupport [ mpi ]
   ++ lib.optionals rocmSupport [ rocmtoolkit_joined ];

From 29501a614d4580b213a0f1500adec0c529a299e2 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Wed, 25 Oct 2023 10:09:43 -0500
Subject: [PATCH 14/19] python3Packages.skrl: fix for python-3.10 and use
 pyproject

---
 pkgs/development/python-modules/skrl/default.nix | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/pkgs/development/python-modules/skrl/default.nix b/pkgs/development/python-modules/skrl/default.nix
index 20f43924b9d08..417c1c1fe3515 100644
--- a/pkgs/development/python-modules/skrl/default.nix
+++ b/pkgs/development/python-modules/skrl/default.nix
@@ -1,9 +1,8 @@
 { lib
 , buildPythonPackage
 , fetchFromGitHub
-, fetchpatch
 , pythonOlder
-, pytestCheckHook
+, setuptools
 , gym
 , gymnasium
 , torch
@@ -11,13 +10,13 @@
 , tqdm
 , wandb
 , packaging
+, pytestCheckHook
 }:
 
 buildPythonPackage rec {
   pname = "skrl";
   version = "1.0.0";
-  format = "setuptools";
-
+  pyproject = true;
   disabled = pythonOlder "3.6";
 
   src = fetchFromGitHub {
@@ -27,14 +26,7 @@ buildPythonPackage rec {
     hash = "sha256-89OoJanmaB74SLF1qMI8WFBdN1czS7Yr7BmojaRdo4M=";
   };
 
-  patches = [
-    # remove after next release:
-    (fetchpatch {
-       name = "fix-python_requires-specification";
-       url = "https://github.com/Toni-SM/skrl/pull/62/commits/9b554adfe2da6cd97cccbbcd418a349cc8f1de80.patch";
-       hash = "sha256-GeASMU1Pgy8U1zaIAVroBDjYaY+n93XP5uFyP4U9lok=";
-    })
-  ];
+  nativeBuildInputs = [ setuptools ];
 
   propagatedBuildInputs = [
     gym

From 8398d16cb0e44bddc5a0ad01a5ec122ec47f53e2 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Thu, 26 Oct 2023 10:27:16 -0500
Subject: [PATCH 15/19] python3Packages.spacy_models: add ja and sl language
 models

---
 .../python-modules/spacy/models.json          | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/pkgs/development/python-modules/spacy/models.json b/pkgs/development/python-modules/spacy/models.json
index 0514c9e41971a..7f25c4281c37b 100644
--- a/pkgs/development/python-modules/spacy/models.json
+++ b/pkgs/development/python-modules/spacy/models.json
@@ -197,6 +197,30 @@
     "sha256": "0dmhv1fa46hi78jgv562v4x3mfl7svchs6kiz35s63ph9ik5r6f2",
     "license": "cc-by-sa-40"
   },
+  {
+    "pname": "ja_core_news_lg",
+    "version": "3.7.0",
+    "sha256": "1nb77kivzy0wixsw8ijmw78fffkpqa63kykqph04jzmh75ra4wvg",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "ja_core_news_md",
+    "version": "3.7.0",
+    "sha256": "0p22bwc24q76cl7ndszvhqgllvq3ws3i3vbjsp5xvhxxls6p49x9",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "ja_core_news_sm",
+    "version": "3.7.0",
+    "sha256": "0bfvkds4dqynjshk2lxfya9yfcnbvwjfhc6n7yh0852ms1ycicaw",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "ja_core_news_trf",
+    "version": "3.7.0",
+    "sha256": "0n2lqql4flnilgf671n5mcdp8vi5pdjfz3vymxsapc1gyp29jykk",
+    "license": "cc-by-sa-30"
+  },
   {
     "pname": "it_core_news_lg",
     "version": "3.7.0",
@@ -377,6 +401,30 @@
     "sha256": "11mh1rd0q024xfagdqkly1n4nndksrlq650n51jl1x1pmzlsdgzl",
     "license": "mit"
   },
+  {
+    "pname": "sl_core_news_lg",
+    "version": "3.7.0",
+    "sha256": "128ayhp21szc31ckiq3i8vib42i9xnz4lpi1709gjdc38cpmpnlp",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "sl_core_news_md",
+    "version": "3.7.0",
+    "sha256": "07gx174gw5q1zgyyg1xhfplihhnr311f9562ri5pdd2hgjyz58yb",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "sl_core_news_sm",
+    "version": "3.7.0",
+    "sha256": "005xwsnh5y3pf8y64lhvrzcbh8y34yr3in204as6hv7krsfg8bxa",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "sl_core_news_trf",
+    "version": "3.7.0",
+    "sha256": "0x97lwm1i2dq4kdg6rvarh9mnlcx45cnwq80qpjwv3b7zmviyq8c",
+    "license": "cc-by-sa-40"
+  },
   {
     "pname": "sv_core_news_lg",
     "version": "3.7.0",

From 6c8a5deaea4dd3ddac146c87b9d4d7a869022f26 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Fri, 27 Oct 2023 11:15:19 -0500
Subject: [PATCH 16/19] python3Packages.cleanlab: fixup build, tests, use
 pyproject

The packages in nativeCheckInputs lead to some nasty duplicate issues in the enclosure

This is worked around by setting catchConflicts to false when doCheck is enabled
---
 .../python-modules/cleanlab/default.nix       | 49 ++++++++++++++-----
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/pkgs/development/python-modules/cleanlab/default.nix b/pkgs/development/python-modules/cleanlab/default.nix
index f1de682c40b53..fdaf384262b8e 100644
--- a/pkgs/development/python-modules/cleanlab/default.nix
+++ b/pkgs/development/python-modules/cleanlab/default.nix
@@ -6,23 +6,27 @@
 , termcolor
 , tqdm
 , pandas
+, setuptools
 # test dependencies
+, pytestCheckHook
+, pytest-lazy-fixture
 , tensorflow
 , torch
 , datasets
 , torchvision
 , keras
 , fasttext
+, hypothesis
+, wget
+, matplotlib
+, skorch
 }:
-let
+
+buildPythonPackage rec {
   pname = "cleanlab";
   version = "2.5.0";
-in
-buildPythonPackage {
-  inherit pname version;
-  format = "setuptools";
-
-  disabled = pythonOlder "3.8";
+  pyproject = true;
+  disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
     owner = "cleanlab";
@@ -31,11 +35,7 @@ buildPythonPackage {
     hash = "sha256-5XQQVrhjpvjwtFM79DqttObmw/GQLkMQVXb5jhiC8e0=";
   };
 
-  # postPatch = ''
-  #   substituteInPlace pyproject.toml \
-  #     --replace '"rich <= 13.0.1"' '"rich"' \
-  #     --replace '"numpy < 1.24.0"' '"numpy"'
-  # '';
+  nativeBuildInputs = [ setuptools ];
 
   propagatedBuildInputs = [
     scikit-learn
@@ -44,13 +44,38 @@ buildPythonPackage {
     pandas
   ];
 
+  # This is ONLY turned off when we have testing enabled.
+  # The reason we do this is because of duplicate packages in the enclosure
+  # when using the packages in nativeCheckInputs.
+  # Affected packages: grpcio protobuf tensorboard tensorboard-plugin-profile
+  catchConflicts = (!doCheck);
+  doCheck = true;
+
   nativeCheckInputs = [
+    pytestCheckHook
+    pytest-lazy-fixture
     tensorflow
     torch
     datasets
     torchvision
     keras
     fasttext
+    hypothesis
+    wget
+    matplotlib
+    skorch
+  ];
+
+  disabledTests = [
+    # Requires the datasets we prevent from downloading
+    "test_create_imagelab"
+  ];
+
+  disabledTestPaths = [
+    # Requires internet
+    "tests/test_dataset.py"
+    # Requires the datasets we just prevented from downloading
+    "tests/datalab/test_cleanvision_integration.py"
   ];
 
   meta = with lib; {

From 96619a098441a68fa1ba0039223a6f603be1f468 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Fri, 27 Oct 2023 11:54:53 -0500
Subject: [PATCH 17/19] python3Packages.omegaconf: improve python 3.11 support

This is mainly for fixing fairseq's python 3.11 build
---
 ...support-for-dataclasses_missing_type.patch | 21 +++++++++++++++++++
 .../python-modules/omegaconf/default.nix      |  8 +++++--
 2 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 pkgs/development/python-modules/omegaconf/0000-add-support-for-dataclasses_missing_type.patch

diff --git a/pkgs/development/python-modules/omegaconf/0000-add-support-for-dataclasses_missing_type.patch b/pkgs/development/python-modules/omegaconf/0000-add-support-for-dataclasses_missing_type.patch
new file mode 100644
index 0000000000000..6435d86a68a29
--- /dev/null
+++ b/pkgs/development/python-modules/omegaconf/0000-add-support-for-dataclasses_missing_type.patch
@@ -0,0 +1,21 @@
+diff --git a/omegaconf/omegaconf.py b/omegaconf/omegaconf.py
+index efde14a..a2a050e 100644
+--- a/omegaconf/omegaconf.py
++++ b/omegaconf/omegaconf.py
+@@ -7,6 +7,7 @@ import pathlib
+ import sys
+ import warnings
+ from collections import defaultdict
++from dataclasses import _MISSING_TYPE
+ from contextlib import contextmanager
+ from enum import Enum
+ from textwrap import dedent
+@@ -828,6 +829,8 @@ class OmegaConf:
+ 
+             if obj is _DEFAULT_MARKER_:
+                 obj = {}
++            if isinstance(obj, _MISSING_TYPE):
++                return OmegaConf.create({}, parent=parent, flags=flags)
+             if isinstance(obj, str):
+                 obj = yaml.load(obj, Loader=get_yaml_loader())
+                 if obj is None:
diff --git a/pkgs/development/python-modules/omegaconf/default.nix b/pkgs/development/python-modules/omegaconf/default.nix
index d506e06d6f671..c6dfada82bd28 100644
--- a/pkgs/development/python-modules/omegaconf/default.nix
+++ b/pkgs/development/python-modules/omegaconf/default.nix
@@ -3,6 +3,7 @@
 , antlr4-python3-runtime
 , buildPythonPackage
 , fetchFromGitHub
+, setuptools
 , jre_minimal
 , pydevd
 , pytest-mock
@@ -15,8 +16,7 @@
 buildPythonPackage rec {
   pname = "omegaconf";
   version = "2.3.0";
-  format = "setuptools";
-
+  pyproject = true;
   disabled = pythonOlder "3.6";
 
   src = fetchFromGitHub {
@@ -31,6 +31,9 @@ buildPythonPackage rec {
       src = ./antlr4.patch;
       antlr_jar = "${antlr4.out}/share/java/antlr-${antlr4.version}-complete.jar";
     })
+
+    # https://github.com/omry/omegaconf/pull/1137
+    ./0000-add-support-for-dataclasses_missing_type.patch
   ];
 
   postPatch = ''
@@ -42,6 +45,7 @@ buildPythonPackage rec {
   '';
 
   nativeBuildInputs = [
+    setuptools
     jre_minimal
   ];
 

From 77ab93a3eecc7dca87b4368597399be3a96c2f1f Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Fri, 27 Oct 2023 12:23:07 -0500
Subject: [PATCH 18/19] python3Packages.fairseq: fix python 3.11 build

---
 .../python-modules/fairseq/default.nix        | 26 +++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/pkgs/development/python-modules/fairseq/default.nix b/pkgs/development/python-modules/fairseq/default.nix
index 92d8a5e78560f..35275c32780a1 100644
--- a/pkgs/development/python-modules/fairseq/default.nix
+++ b/pkgs/development/python-modules/fairseq/default.nix
@@ -2,6 +2,7 @@
 , buildPythonPackage
 , pythonOlder
 , fetchFromGitHub
+, fetchpatch
 
 # Native build inputs
 , cython
@@ -27,12 +28,12 @@
 , hypothesis
 , pytestCheckHook
 }:
-let
+
+buildPythonPackage rec {
   pname = "fairseq";
   version = "0.12.3";
-in
-buildPythonPackage rec {
-  inherit version pname;
+  pyproject = true;
+  disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
     owner = "pytorch";
@@ -41,17 +42,25 @@ buildPythonPackage rec {
     hash = "sha256-XX/grU5ljQCwx33miGoFc/7Uj9fZDtmhm4Fz7L4U+Bc=";
   };
 
-  disabled = pythonOlder "3.7";
+  patches = [
+    # https://github.com/facebookresearch/fairseq/pull/5359
+    (fetchpatch {
+      url = "https://github.com/facebookresearch/fairseq/commit/2fa0768c2115b0a4c207cfa3e1b3e4ff3ad9a00c.patch";
+      hash = "sha256-aYYP/knQX6q6vhyA6q9uOOYfRhDAuJCo9QJWfFEDuuA=";
+    })
+  ];
 
   nativeBuildInputs = [
     cython
     pythonRelaxDepsHook
     which
   ];
+
   pythonRelaxDeps = [
     "hydra-core"
     "omegaconf"
   ];
+
   propagatedBuildInputs = [
     cffi
     hydra-core
@@ -74,6 +83,7 @@ buildPythonPackage rec {
   ];
 
   pythonImportsCheck = [ "fairseq" ];
+
   preCheck = ''
     export HOME=$TMPDIR
     cd tests
@@ -82,6 +92,7 @@ buildPythonPackage rec {
   pytestFlagsArray = [
     "--import-mode append"
   ];
+
   disabledTests = [
     # this test requires xformers
     "test_xformers_single_forward_parity"
@@ -96,6 +107,11 @@ buildPythonPackage rec {
     "test_librispeech_s2t_conformer_s_checkpoint"
   ];
 
+  disabledTestPaths = [
+    # ValueError: mutable default ... for field bar is not allowed: use default_factory
+    "test_dataclass_utils.py"
+  ];
+
   meta = with lib; {
     description = "Facebook AI Research Sequence-to-Sequence Toolkit";
     homepage = "https://github.com/pytorch/fairseq";

From 3aab2bc4192499d1ad51a6aa0d79779b5044a8c5 Mon Sep 17 00:00:00 2001
From: Madoura <madouura@gmail.com>
Date: Fri, 3 Nov 2023 13:15:37 -0500
Subject: [PATCH 19/19] python3Packages.torchmetrics: work around tests
 duplicate package issue

The repo was moved from PyTorchLightning/metrics to Lightning-AI/torchmetrics
---
 .../python-modules/torchmetrics/default.nix   | 31 +++++++++++++------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/pkgs/development/python-modules/torchmetrics/default.nix b/pkgs/development/python-modules/torchmetrics/default.nix
index 85e85c16afc79..ee66ee817d1f7 100644
--- a/pkgs/development/python-modules/torchmetrics/default.nix
+++ b/pkgs/development/python-modules/torchmetrics/default.nix
@@ -14,6 +14,8 @@
 , pytestCheckHook
 , torchmetrics
 , pytorch-lightning
+, pytest-doctestplus
+, pytest-xdist
 }:
 
 let
@@ -22,10 +24,11 @@ let
 in
 buildPythonPackage {
   inherit pname version;
+  pyproject = true;
 
   src = fetchFromGitHub {
-    owner = "PyTorchLightning";
-    repo = "metrics";
+    owner = "Lightning-AI";
+    repo = "torchmetrics";
     rev = "refs/tags/v${version}";
     hash = "sha256-g5JuTbiRd8yWx2nM3UE8ejOhuZ0XpAQdS5AC9AlrSFY=";
   };
@@ -51,22 +54,33 @@ buildPythonPackage {
     cloudpickle
     psutil
     pytestCheckHook
+    pytest-doctestplus
+    pytest-xdist
   ];
 
   # A cyclic dependency in: integrations/test_lightning.py
   doCheck = false;
   passthru.tests.check = torchmetrics.overridePythonAttrs (_: {
+    pname = "${pname}-check";
     doCheck = true;
+    # We don't have to install because the only purpose
+    # of this passthru test is to, well, test.
+    # This fixes having to set `catchConflicts` to false.
+    dontInstall = true;
   });
 
+  disabledTests = [
+    # `IndexError: list index out of range`
+    "test_metric_lightning_log"
+  ];
+
   disabledTestPaths = [
     # These require too many "leftpad-level" dependencies
-    "tests/text"
-    "tests/audio"
-    "tests/image"
+    # Also too cross-dependent
+    "tests/unittests"
 
-    # A few non-deterministic things like test_check_compute_groups_is_faster
-    "tests/bases/test_collections.py"
+    # A trillion import path mismatch errors
+    "src/torchmetrics"
   ];
 
   pythonImportsCheck = [
@@ -75,11 +89,10 @@ buildPythonPackage {
 
   meta = with lib; {
     description = "Machine learning metrics for distributed, scalable PyTorch applications (used in pytorch-lightning)";
-    homepage = "https://torchmetrics.readthedocs.io";
+    homepage = "https://lightning.ai/docs/torchmetrics/";
     license = licenses.asl20;
     maintainers = with maintainers; [
       SomeoneSerge
     ];
   };
 }
-