Unverified Commit 6fa9fea1 authored by Someone's avatar Someone Committed by GitHub
Browse files

Merge pull request #329836 from GaetanLepage/torch 

python311Packages.{torch,torch-bin}: 2.3.1 -> 2.4.0
parents 5660b83b e100b56c

pkgs/development/python-modules/torch/bin.nix

+5 −3
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ let 
  pyVerNoDot = builtins.replaceStrings [ "." ] [ "" ] python.pythonVersion;

  srcs = import ./binary-hashes.nix version;

  unsupported = throw "Unsupported system";

  version = "2.3.1";

  version = "2.4.0";

in

buildPythonPackage {

  inherit version;
@@ -102,7 +102,9 @@ buildPythonPackage { 
  #

  # This is a quick hack to add `libnvrtc` to the runpath so that torch can find

  # it when it is needed at runtime.

  extraRunpaths = lib.optionals stdenv.hostPlatform.isLinux [ "${lib.getLib cudaPackages.cuda_nvrtc}/lib" ];

  extraRunpaths = lib.optionals stdenv.hostPlatform.isLinux [

    "${lib.getLib cudaPackages.cuda_nvrtc}/lib"

  ];

  postPhases = lib.optionals stdenv.isLinux [ "postPatchelfPhase" ];

  postPatchelfPhase = ''

    while IFS= read -r -d $'\0' elf ; do

pkgs/development/python-modules/torch/binary-hashes.nix

+46 −46
Original line number Diff line number Diff line
@@ -7,81 +7,81 @@ 


version:

builtins.getAttr version {

  "2.3.1" = {

  "2.4.0" = {

    x86_64-linux-38 = {

      name = "torch-2.3.1-cp38-cp38-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.3.1%2Bcu121-cp38-cp38-linux_x86_64.whl";

      hash = "sha256-TkEPNC/YbHO+oO0kVQnV/15od72lSySfdaM9U1yHfy8=";

      name = "torch-2.4.0-cp38-cp38-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.4.0%2Bcu121-cp38-cp38-linux_x86_64.whl";

      hash = "sha256-ikebcXQK+SpOG5kEW+qDHz2nMYfIw+PSErHm/jkxWyE=";

    };

    x86_64-linux-39 = {

      name = "torch-2.3.1-cp39-cp39-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.3.1%2Bcu121-cp39-cp39-linux_x86_64.whl";

      hash = "sha256-3+phA2LA4qX/KNMi1qpl1l4D4TNJlhGaWjdwx9GCGsQ=";

      name = "torch-2.4.0-cp39-cp39-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.4.0%2Bcu121-cp39-cp39-linux_x86_64.whl";

      hash = "sha256-iIObriuWfdOFeaqwu6lRDpKxHau8Th3V5jDF/WVfXac=";

    };

    x86_64-linux-310 = {

      name = "torch-2.3.1-cp310-cp310-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.3.1%2Bcu121-cp310-cp310-linux_x86_64.whl";

      hash = "sha256-8N610vkypo7VRiW6FA7dvyryK+l47hm5tjyYat1kJbI=";

      name = "torch-2.4.0-cp310-cp310-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.4.0%2Bcu121-cp310-cp310-linux_x86_64.whl";

      hash = "sha256-KL+6CE3KUqBsRl160PPMNyw1/FA/PquIHMF6X9gpFOc=";

    };

    x86_64-linux-311 = {

      name = "torch-2.3.1-cp311-cp311-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.3.1%2Bcu121-cp311-cp311-linux_x86_64.whl";

      hash = "sha256-kl40rwkFBipItPgtDmZWNBrU1iaDSmqCRe9OruU3XJg=";

      name = "torch-2.4.0-cp311-cp311-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.4.0%2Bcu121-cp311-cp311-linux_x86_64.whl";

      hash = "sha256-qf/zLTZeDHS2kJSAVIsuKRMUogStsptrtvLG0z+L4mw=";

    };

    x86_64-linux-312 = {

      name = "torch-2.3.1-cp312-cp312-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.3.1%2Bcu121-cp312-cp312-linux_x86_64.whl";

      hash = "sha256-s8WG9Ksl6D7//M+5cHnpEyUym8IoFmVVxLuTlXdT1Oo=";

      name = "torch-2.4.0-cp312-cp312-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu121/torch-2.4.0%2Bcu121-cp312-cp312-linux_x86_64.whl";

      hash = "sha256-SaxVpkl93W0M3VG16ifY6+IMknMHeFXpyW6w3CifB8M=";

    };

    aarch64-darwin-38 = {

      name = "torch-2.3.1-cp38-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp38-none-macosx_11_0_arm64.whl";

      hash = "sha256-vuC9M9xYqo/Ip1J4dum5oOgSrQgSIFSlv/LOWr8AWxA=";

      name = "torch-2.4.0-cp38-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp38-none-macosx_11_0_arm64.whl";

      hash = "sha256-OvTeKmGPsGXnhATEuieoGKe3lX6u/yjGxmzn+1BLaLg=";

    };

    aarch64-darwin-39 = {

      name = "torch-2.3.1-cp39-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp39-none-macosx_11_0_arm64.whl";

      hash = "sha256-K7WveAxVvmj+EA/rBSjS7eus4dVcsuNR3nNYCbpzkes=";

      name = "torch-2.4.0-cp39-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp39-none-macosx_11_0_arm64.whl";

      hash = "sha256-iUD8i5ekxh/bXUajaPIfSjpWKheHnpMutRpexiMQyzE=";

    };

    aarch64-darwin-310 = {

      name = "torch-2.3.1-cp310-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp310-none-macosx_11_0_arm64.whl";

      hash = "sha256-fAmpQ2J3hChIS8+ZX2AEsElSEGruDvRf8LS6tIT1SY0=";

      name = "torch-2.4.0-cp310-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp310-none-macosx_11_0_arm64.whl";

      hash = "sha256-aFQYq5NzDvvucVKIIf9UAFWWlw3Ul78DyJIE+34/cd4=";

    };

    aarch64-darwin-311 = {

      name = "torch-2.3.1-cp311-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp311-none-macosx_11_0_arm64.whl";

      hash = "sha256-p91O04itHz1QK/CUU9X+WWx7Eh3n4M+soeIBd4Lpu6w=";

      name = "torch-2.4.0-cp311-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp311-none-macosx_11_0_arm64.whl";

      hash = "sha256-8Wm06m3JOzozMZYR/MR9wUBuTdU5hE3L0t7EwbluFm0=";

    };

    aarch64-darwin-312 = {

      name = "torch-2.3.1-cp312-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp312-none-macosx_11_0_arm64.whl";

      hash = "sha256-PDM9wuvBiVYVFO2gboHfIr+Ptk4jhHRrLLnwT5bR1Mg=";

      name = "torch-2.4.0-cp312-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp312-none-macosx_11_0_arm64.whl";

      hash = "sha256-karwC/4f+kTcW1KAnZqVEp/KECEuyjrCZCDrEXJ8Yog=";

    };

    aarch64-linux-38 = {

      name = "torch-2.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-O3wUmPkE9n6x4zHy6+h0J3GiznG57pvAHelnJX6IHH0=";

      name = "torch-2.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-nrqD+Kj5hUL5F+OQAMkD8VRlWs9jdcBzz81DBqFU64A=";

    };

    aarch64-linux-39 = {

      name = "torch-2.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-22v/S6YnO1muRD3gS1rcNtakC7KJiGYTO/8tUvJ26v4=";

      name = "torch-2.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-J4akfI2N7BdvxnnSqrmm9UnCVFJRC0llCrE0E1JmujM=";

    };

    aarch64-linux-310 = {

      name = "torch-2.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-ZUT98pAYZowKbUobzJVZgsGtpwgGKBsBDLqTvc+9zyI=";

      name = "torch-2.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-fBWeidTs8IQD+dE3PVVEIiQLmxFGoKGRKQadw1enKys=";

    };

    aarch64-linux-311 = {

      name = "torch-2.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-Kq8ON3NMvF/mv8yBraNuy7iZ1N2+E0mL2EqsqKkchig=";

      name = "torch-2.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-OBacsPHmcnw9rI2si5pI0HKkn2Q7kIqZFV7x2Bthves=";

    };

    aarch64-linux-312 = {

      name = "torch-2.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-d9LeGklaHAf1ksM4oNWS5VzAudL4ADCeRqDqLA46KRk=";

      name = "torch-2.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";

      hash = "sha256-7IY1E1C716vqNDb6b0Iws7C793oibkS6RhhA4gT8zXE=";

    };

  };

}

pkgs/development/python-modules/torch/default.nix

+21 −11
Original line number Diff line number Diff line
@@ -99,7 +99,7 @@ let 


  setBool = v: if v then "1" else "0";



  # https://github.com/pytorch/pytorch/blob/v2.0.1/torch/utils/cpp_extension.py#L1744

  # https://github.com/pytorch/pytorch/blob/v2.4.0/torch/utils/cpp_extension.py#L1953

  supportedTorchCudaCapabilities =

    let

      real = [
@@ -119,6 +119,7 @@ let 
        "8.7"

        "8.9"

        "9.0"

        "9.0a"

      ];

      ptx = lists.map (x: "${x}+PTX") real;

    in
@@ -206,13 +207,14 @@ let 
    # In particular, this triggered warnings from cuda's `aliases.nix`

    "Magma cudaPackages does not match cudaPackages" =

      cudaSupport && (effectiveMagma.cudaPackages.cudaVersion != cudaPackages.cudaVersion);

    "Rocm support is currently broken because `rocmPackages.hipblaslt` is unpackaged. (2024-06-09)" = rocmSupport;

    "Rocm support is currently broken because `rocmPackages.hipblaslt` is unpackaged. (2024-06-09)" =

      rocmSupport;

  };

in

buildPythonPackage rec {

  pname = "torch";

  # Don't forget to update torch-bin to the same version.

  version = "2.3.1";

  version = "2.4.0";

  pyproject = true;



  disabled = pythonOlder "3.8.0";
@@ -230,11 +232,16 @@ buildPythonPackage rec { 
    repo = "pytorch";

    rev = "refs/tags/v${version}";

    fetchSubmodules = true;

    hash = "sha256-vpgtOqzIDKgRuqdT8lB/g6j+oMIH1RPxdbjtlzZFjV8=";

    hash = "sha256-s49rtarGNNFpnNG+kfJtZLE8ND53Ma201I0cOjeFSts=";

  };



  patches =

    lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]

    [

      # Allow setting PYTHON_LIB_REL_PATH with an environment variable.

      # https://github.com/pytorch/pytorch/pull/128419

      ./passthrough-python-lib-rel-path.patch

    ]

    ++ lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]

    ++ lib.optionals (stdenv.isDarwin && stdenv.isx86_64) [

      # pthreadpool added support for Grand Central Dispatch in April

      # 2020. However, this relies on functionality (DISPATCH_APPLY_AUTO)
@@ -279,11 +286,12 @@ buildPythonPackage rec { 
          'message(FATAL_ERROR "Found NCCL header version and library version' \

          'message(WARNING "Found NCCL header version and library version'

    ''

    # Remove PyTorch's FindCUDAToolkit.cmake and to use CMake's default.

    # We do not remove the entirety of cmake/Modules_CUDA_fix because we need FindCUDNN.cmake.

    # Remove PyTorch's FindCUDAToolkit.cmake and use CMake's default.

    # NOTE: Parts of pytorch rely on unmaintained FindCUDA.cmake with custom patches to support e.g.

    # newer architectures (sm_90a). We do want to delete vendored patches, but have to keep them

    # until https://github.com/pytorch/pytorch/issues/76082 is addressed

    + lib.optionalString cudaSupport ''

      rm cmake/Modules/FindCUDAToolkit.cmake

      rm -rf cmake/Modules_CUDA_fix/{upstream,FindCUDA.cmake}

    ''

    # error: no member named 'aligned_alloc' in the global namespace; did you mean simply 'aligned_alloc'

    # This lib overrided aligned_alloc hence the error message. Tltr: his function is linkable but not in header.
@@ -375,6 +383,10 @@ buildPythonPackage rec { 
  USE_SYSTEM_NCCL = USE_NCCL;

  USE_STATIC_NCCL = USE_NCCL;



  # Set the correct Python library path, broken since

  # https://github.com/pytorch/pytorch/commit/3d617333e

  PYTHON_LIB_REL_PATH = "${placeholder "out"}/${python.sitePackages}";



  # Suppress a weird warning in mkl-dnn, part of ideep in pytorch

  # (upstream seems to have fixed this in the wrong place?)

  # https://github.com/intel/mkl-dnn/commit/8134d346cdb7fe1695a2aa55771071d455fae0bc
@@ -466,9 +478,7 @@ buildPythonPackage rec { 
        libcusolver

        libcusparse

      ]

      ++ lists.optionals (cudaPackages ? cudnn) [

        cudnn

      ]

      ++ lists.optionals (cudaPackages ? cudnn) [ cudnn ]

      ++ lists.optionals useSystemNccl [

        # Some platforms do not support NCCL (i.e., Jetson)

        nccl # Provides nccl.h AND a static copy of NCCL!

pkgs/development/python-modules/torch/fix-cmake-cuda-toolkit.patch

+12 −48
Original line number Diff line number Diff line 
diff --git a/CMakeLists.txt b/CMakeLists.txt

index 9194e520bb0..d05fdcfb6cb 100644

index c4cd4b2c2a..e983b21353 100644

--- a/CMakeLists.txt

+++ b/CMakeLists.txt

@@ -1160,10 +1160,6 @@ if(BUILD_SHARED_LIBS)

       ${PROJECT_SOURCE_DIR}/cmake/Modules_CUDA_fix

@@ -1319,10 +1319,6 @@ if(BUILD_SHARED_LIBS)

     DIRECTORY ${PROJECT_SOURCE_DIR}/cmake/Modules_CUDA_fix

     DESTINATION share/cmake/Caffe2/

     COMPONENT dev)

-  install(FILES

-      ${PROJECT_SOURCE_DIR}/cmake/Modules/FindCUDAToolkit.cmake

-  install(

-    FILES ${PROJECT_SOURCE_DIR}/cmake/Modules/FindCUDAToolkit.cmake

-    DESTINATION share/cmake/Caffe2/

-    COMPONENT dev)

   install(FILES

       ${PROJECT_SOURCE_DIR}/cmake/Modules/FindCUSPARSELT.cmake

   install(

     FILES ${PROJECT_SOURCE_DIR}/cmake/Modules/FindCUSPARSELT.cmake

     DESTINATION share/cmake/Caffe2/
 
diff --git a/cmake/public/cuda.cmake b/cmake/public/cuda.cmake

index c7595774d81..4fc43771810 100644

--- a/cmake/public/cuda.cmake

+++ b/cmake/public/cuda.cmake

@@ -61,9 +61,15 @@ find_package(CUDAToolkit REQUIRED)

 cmake_policy(POP)

 

 if(NOT CMAKE_CUDA_COMPILER_VERSION VERSION_EQUAL CUDAToolkit_VERSION)

-  message(FATAL_ERROR "Found two conflicting CUDA versions:\n"

-                      "V${CMAKE_CUDA_COMPILER_VERSION} in '${CUDA_INCLUDE_DIRS}' and\n"

-                      "V${CUDAToolkit_VERSION} in '${CUDAToolkit_INCLUDE_DIRS}'")

+  if(CUDA_INCLUDE_DIRS IN_LIST CUDAToolkit_INCLUDE_DIR)

+    message(STATUS "CUDA_INCLUDE_DIRS is a substring of CUDAToolkit_INCLUDE_DIR. "

+                   "Setting CUDA_INCLUDE_DIRS to CUDAToolkit_INCLUDE_DIR.")

+    set(CUDA_INCLUDE_DIRS "${CUDAToolkit_INCLUDE_DIR}")

+  else()

+    message(FATAL_ERROR "Found two conflicting CUDA installs:\n"

+                        "V${CMAKE_CUDA_COMPILER_VERSION} in '${CUDA_INCLUDE_DIRS}' and\n"

+                        "V${CUDAToolkit_VERSION} in '${CUDAToolkit_INCLUDE_DIR}'")

+  endif()

 endif()

 

 if(NOT TARGET CUDA::nvToolsExt)

diff --git a/tools/setup_helpers/cmake.py b/tools/setup_helpers/cmake.py

index fb19b66dfba..3f83bef32fe 100644

--- a/tools/setup_helpers/cmake.py

+++ b/tools/setup_helpers/cmake.py

@@ -207,6 +207,8 @@ class CMake:

                     "BUILDING_WITH_TORCH_LIBS",

                     "CUDA_HOST_COMPILER",

                     "CUDA_NVCC_EXECUTABLE",

+                    "CUDAToolkit_ROOT",

+                    "CUDAToolkit_INCLUDE_DIR",

                     "CUDA_SEPARABLE_COMPILATION",

                     "CUDNN_LIBRARY",

                     "CUDNN_INCLUDE_DIR",

pkgs/development/python-modules/torch/passthrough-python-lib-rel-path.patch

0 → 100644
+12 −0
Original line number Diff line number Diff line 
diff --git a/tools/setup_helpers/cmake.py b/tools/setup_helpers/cmake.py

index 5481ce46031c..d50d9d547399 100644

--- a/tools/setup_helpers/cmake.py

+++ b/tools/setup_helpers/cmake.py

@@ -231,6 +231,7 @@ def generate(

                     "SELECTED_OP_LIST",

                     "TORCH_CUDA_ARCH_LIST",

                     "TRACING_BASED",

+                    "PYTHON_LIB_REL_PATH",

                 )

             }

         )