Commit ad01be00 authored by Junji Hashimoto's avatar Junji Hashimoto Committed by Gaetan Lepage
Browse files

python3Packages.torch-bin: 1.13.1 -> 2.0.0

parent 8c168726

pkgs/development/python-modules/torch/bin.nix

+47 −14
Original line number Diff line number Diff line 
{ lib, stdenv

, buildPythonPackage

, fetchurl

, isPy37

, isPy38

, isPy39

, isPy310

, python

, pythonAtLeast

, pythonOlder

, addOpenGLRunpath

, cudaPackages

, future

, numpy

, autoPatchelfHook

, patchelf

, pyyaml

, requests

, setuptools

, typing-extensions

, sympy

, jinja2

, networkx

, filelock

, openai-triton

}:



let

  pyVerNoDot = builtins.replaceStrings [ "." ] [ "" ] python.pythonVersion;

  srcs = import ./binary-hashes.nix version;

  unsupported = throw "Unsupported system";

  version = "1.13.1";

  version = "2.0.0";

in buildPythonPackage {

  inherit version;
@@ -29,15 +34,31 @@ in buildPythonPackage { 


  format = "wheel";



  disabled = !(isPy38 || isPy39 || isPy310);

  disabled = (pythonOlder "3.8") || (pythonAtLeast "3.12");



  src = fetchurl srcs."${stdenv.system}-${pyVerNoDot}" or unsupported;



  nativeBuildInputs = [

    addOpenGLRunpath

    autoPatchelfHook

    cudaPackages.autoAddOpenGLRunpathHook

    patchelf

  ];



  buildInputs = with cudaPackages; [

    # $out/${sitePackages}/nvfuser/_C*.so wants libnvToolsExt.so.1 but torch/lib only ships

    # libnvToolsExt-$hash.so.1

    cuda_nvtx

  ];



  autoPatchelfIgnoreMissingDeps = [

    # This is the hardware-dependent userspace driver that comes from

    # nvidia_x11 package. It must be deployed at runtime in

    # /run/opengl-driver/lib or pointed at by LD_LIBRARY_PATH variable, rather

    # than pinned in runpath

    "libcuda.so.1"

  ];



  propagatedBuildInputs = [

    future

    numpy
@@ -45,6 +66,12 @@ in buildPythonPackage { 
    requests

    setuptools

    typing-extensions

    sympy

    jinja2

    networkx

    filelock

  ] ++ lib.optionals stdenv.isx86_64 [

    openai-triton

  ];



  postInstall = ''
@@ -52,14 +79,20 @@ in buildPythonPackage { 
    rm -rf $out/bin

  '';



  postFixup = let

    rpath = lib.makeLibraryPath [ stdenv.cc.cc.lib ];

  in ''

    find $out/${python.sitePackages}/torch/lib -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do

      echo "setting rpath for $lib..."

      patchelf --set-rpath "${rpath}:$out/${python.sitePackages}/torch/lib" "$lib"

      addOpenGLRunpath "$lib"

  postFixup = ''

    addAutoPatchelfSearchPath "$out/${python.sitePackages}/torch/lib"



    patchelf $out/${python.sitePackages}/torch/lib/libcudnn.so.8 --add-needed libcudnn_cnn_infer.so.8



    pushd $out/${python.sitePackages}/torch/lib || exit 1

      for LIBNVRTC in ./libnvrtc*

      do

        case "$LIBNVRTC" in

          ./libnvrtc-builtins*) true;;

          ./libnvrtc*) patchelf "$LIBNVRTC" --add-needed libnvrtc-builtins* ;;

        esac

      done

    popd || exit 1

  '';



  # The wheel-binary is not stripped to avoid the error of `ImportError: libtorch_cuda_cpp.so: ELF load command address/offset not properly aligned.`.

pkgs/development/python-modules/torch/binary-hashes.nix

+57 −37
Original line number Diff line number Diff line
@@ -6,66 +6,86 @@ 
# To add a new version, run "prefetch.sh 'new-version'" to paste the generated file as follows.



version : builtins.getAttr version {

  "1.13.1" = {

  "2.0.0" = {

    x86_64-linux-38 = {

      name = "torch-1.13.1-cp38-cp38-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu117/torch-1.13.1%2Bcu117-cp38-cp38-linux_x86_64.whl";

      hash = "sha256-u/lUbw0Ni1EmPKR5Y3tCaogzX8oANPQs7GPU0y3uBa8=";

      name = "torch-2.0.0-cp38-cp38-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu118/torch-2.0.0%2Bcu118-cp38-cp38-linux_x86_64.whl";

      hash = "sha256-H4766/y7fsOWL9jHw74CxmZu/1OhIEMAanSdZHZWFj4=";

    };

    x86_64-linux-39 = {

      name = "torch-1.13.1-cp39-cp39-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu117/torch-1.13.1%2Bcu117-cp39-cp39-linux_x86_64.whl";

      hash = "sha256-s6wTng1KCzA8wW9R63cUbsfRTAsecCrWOGE2KPUIavc=";

      name = "torch-2.0.0-cp39-cp39-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu118/torch-2.0.0%2Bcu118-cp39-cp39-linux_x86_64.whl";

      hash = "sha256-6rl6n+WefjHWVisYb0NecXsd8zMcrcd25sBzIjmp7Tk=";

    };

    x86_64-linux-310 = {

      name = "torch-1.13.1-cp310-cp310-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu117/torch-1.13.1%2Bcu117-cp310-cp310-linux_x86_64.whl";

      hash = "sha256-FMXJ2wnfjPGzlCo0ecd52m4pOoShYtimrHHiveMOMMU=";

      name = "torch-2.0.0-cp310-cp310-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu118/torch-2.0.0%2Bcu118-cp310-cp310-linux_x86_64.whl";

      hash = "sha256-S2kOK3fyEHNQDGXYu56pZWuMtOlp81c3C7yZKjsHR2Q=";

    };

    x86_64-linux-311 = {

      name = "torch-2.0.0-cp311-cp311-linux_x86_64.whl";

      url = "https://download.pytorch.org/whl/cu118/torch-2.0.0%2Bcu118-cp311-cp311-linux_x86_64.whl";

      hash = "sha256-I4Vz02LFZBE0UQRvZwjDuBWP5rG39sA7cnMyfZVd61Q=";

    };

    x86_64-darwin-38 = {

      name = "torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl";

      hash = "sha256-M+Z+6lJuC7uRUSY+ZUF6nvLY+lPL5ijocxAGDJ3PoxI=";

      name = "torch-2.0.0-cp38-none-macosx_10_9_x86_64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp38-none-macosx_10_9_x86_64.whl";

      hash = "sha256-zHiMu7vG60yQ5SxVDv0GdYbCaTCSzzZ8E1s0iTpkrng=";

    };

    x86_64-darwin-39 = {

      name = "torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl";

      hash = "sha256-aTB5HvqHV8tpdK9z1Jlra1DFkogqMkuPsFicapui3a8=";

      name = "torch-2.0.0-cp39-none-macosx_10_9_x86_64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp39-none-macosx_10_9_x86_64.whl";

      hash = "sha256-bguXvrA3oWVmnDElkfJCOC6RCaJA4gBU1aV4LZI2ytA=";

    };

    x86_64-darwin-310 = {

      name = "torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl";

      hash = "sha256-OTpic8gy4EdYEGP7dDNf9QtMVmIXAZzGrOMYzXnrBWY=";

      name = "torch-2.0.0-cp310-none-macosx_10_9_x86_64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp310-none-macosx_10_9_x86_64.whl";

      hash = "sha256-zptaSb1RPf95UKWgfW4mWU3VGYnO4FujiLA+jjZv1dU=";

    };

    x86_64-darwin-311 = {

      name = "torch-2.0.0-cp311-none-macosx_10_9_x86_64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp311-none-macosx_10_9_x86_64.whl";

      hash = "sha256-AYWGIPJfJeep7EtUf/OOXifJLTjsTMupz7+zHXBx7Zw=";

    };

    aarch64-darwin-38 = {

      name = "torch-1.13.1-cp38-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-1.13.1-cp38-none-macosx_11_0_arm64.whl";

      hash = "sha256-7usgTTD9QK9qLYCHm0an77489Dzb64g43U89EmzJCys=";

      name = "torch-2.0.0-cp38-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp38-none-macosx_11_0_arm64.whl";

      hash = "sha256-0pJkDw/XK3oxsqbjtjXrUGX8vt1EePnK0aHnqeyGHTU=";

    };

    aarch64-darwin-39 = {

      name = "torch-1.13.1-cp39-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-1.13.1-cp39-none-macosx_11_0_arm64.whl";

      hash = "sha256-4N+QKnx91seVaYUy7llwzomGcmJWNdiF6t6ZduWgSUk=";

      name = "torch-2.0.0-cp39-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp39-none-macosx_11_0_arm64.whl";

      hash = "sha256-KXpJGa/xwPmKWOvpaSAPcTUKHU1PmG2/1gwC/854Dpk=";

    };

    aarch64-darwin-310 = {

      name = "torch-1.13.1-cp310-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-1.13.1-cp310-none-macosx_11_0_arm64.whl";

      hash = "sha256-ASKAaxEblJ0h+hpfl2TR/S/MSkfLf4/5FCBP1Px1LtU=";

      name = "torch-2.0.0-cp310-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp310-none-macosx_11_0_arm64.whl";

      hash = "sha256-U+HDPGiWWDzbmlg2k+IumSZkRMSkM5Ld3FYmQNOeVCs=";

    };

    aarch64-darwin-311 = {

      name = "torch-2.0.0-cp311-none-macosx_11_0_arm64.whl";

      url = "https://download.pytorch.org/whl/cpu/torch-2.0.0-cp311-none-macosx_11_0_arm64.whl";

      hash = "sha256-mi5TtXg+9YlqavM4s214LyjoPI3fwqxEtnsGbZ129Jg=";

    };

    aarch64-linux-38 = {

      name = "torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl";

      hash = "sha256-34Q0sGlenOuMxwZQr8ExDYupSebbKgUl3dnDsrGB5f4=";

      name = "torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl";

      hash = "sha256-EbA4T+PBjAG4/FmS5w/FGc3mXkTFHMh74YOMGAPa9C8=";

    };

    aarch64-linux-39 = {

      name = "torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl";

      hash = "sha256-LDWBo/2B6x8PIpl83f/qVp/qU7r6NyssBHHbNzsmqvw=";

      name = "torch-2.0.0-cp39-cp39-manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/torch-2.0.0-cp39-cp39-manylinux2014_aarch64.whl";

      hash = "sha256-qDsmvWrjb79f7j1Wlz2YFuIALoo7fZIFUxFnwoqqOKc=";

    };

    aarch64-linux-310 = {

      name = "torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl";

      hash = "sha256-2f54XTdfLial1eul3pH4nmo75dEe+0l+dnBf35P6PC4=";

      name = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl";

      hash = "sha256-nwH+H2Jj8xvQThdXlG/WOtUxrjfyi7Lb9m9cgm7gifQ=";

    };

    aarch64-linux-311 = {

      name = "torch-2.0.0-cp311-cp311-manylinux2014_aarch64.whl";

      url = "https://download.pytorch.org/whl/torch-2.0.0-cp311-cp311-manylinux2014_aarch64.whl";

      hash = "sha256-1Dmuw0nJjxKBnoVkuMVACORhPdRChYKvDm4UwkyoWHA=";

    };

  };

}

pkgs/development/python-modules/torch/prefetch.sh

+5 −1
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ set -eou pipefail 


version=$1



linux_cuda_version="cu117"

linux_cuda_version="cu118"

linux_cuda_bucket="https://download.pytorch.org/whl/${linux_cuda_version}"

linux_cpu_bucket="https://download.pytorch.org/whl"

darwin_bucket="https://download.pytorch.org/whl/cpu"
@@ -14,15 +14,19 @@ url_and_key_list=( 
  "x86_64-linux-38 $linux_cuda_bucket/torch-${version}%2B${linux_cuda_version}-cp38-cp38-linux_x86_64.whl torch-${version}-cp38-cp38-linux_x86_64.whl"

  "x86_64-linux-39 $linux_cuda_bucket/torch-${version}%2B${linux_cuda_version}-cp39-cp39-linux_x86_64.whl torch-${version}-cp39-cp39-linux_x86_64.whl"

  "x86_64-linux-310 $linux_cuda_bucket/torch-${version}%2B${linux_cuda_version}-cp310-cp310-linux_x86_64.whl torch-${version}-cp310-cp310-linux_x86_64.whl"

  "x86_64-linux-311 $linux_cuda_bucket/torch-${version}%2B${linux_cuda_version}-cp311-cp311-linux_x86_64.whl torch-${version}-cp311-cp311-linux_x86_64.whl"

  "x86_64-darwin-38 $darwin_bucket/torch-${version}-cp38-none-macosx_10_9_x86_64.whl torch-${version}-cp38-none-macosx_10_9_x86_64.whl"

  "x86_64-darwin-39 $darwin_bucket/torch-${version}-cp39-none-macosx_10_9_x86_64.whl torch-${version}-cp39-none-macosx_10_9_x86_64.whl"

  "x86_64-darwin-310 $darwin_bucket/torch-${version}-cp310-none-macosx_10_9_x86_64.whl torch-${version}-cp310-none-macosx_10_9_x86_64.whl"

  "x86_64-darwin-311 $darwin_bucket/torch-${version}-cp311-none-macosx_10_9_x86_64.whl torch-${version}-cp311-none-macosx_10_9_x86_64.whl"

  "aarch64-darwin-38 $darwin_bucket/torch-${version}-cp38-none-macosx_11_0_arm64.whl torch-${version}-cp38-none-macosx_11_0_arm64.whl"

  "aarch64-darwin-39 $darwin_bucket/torch-${version}-cp39-none-macosx_11_0_arm64.whl torch-${version}-cp39-none-macosx_11_0_arm64.whl"

  "aarch64-darwin-310 $darwin_bucket/torch-${version}-cp310-none-macosx_11_0_arm64.whl torch-${version}-cp310-none-macosx_11_0_arm64.whl"

  "aarch64-darwin-311 $darwin_bucket/torch-${version}-cp311-none-macosx_11_0_arm64.whl torch-${version}-cp311-none-macosx_11_0_arm64.whl"

  "aarch64-linux-38 $linux_cpu_bucket/torch-${version}-cp38-cp38-manylinux2014_aarch64.whl torch-${version}-cp38-cp38-manylinux2014_aarch64.whl"

  "aarch64-linux-39 $linux_cpu_bucket/torch-${version}-cp39-cp39-manylinux2014_aarch64.whl torch-${version}-cp39-cp39-manylinux2014_aarch64.whl"

  "aarch64-linux-310 $linux_cpu_bucket/torch-${version}-cp310-cp310-manylinux2014_aarch64.whl torch-${version}-cp310-cp310-manylinux2014_aarch64.whl"

  "aarch64-linux-311 $linux_cpu_bucket/torch-${version}-cp311-cp311-manylinux2014_aarch64.whl torch-${version}-cp311-cp311-manylinux2014_aarch64.whl"

)



hashfile="binary-hashes-$version.nix"

pkgs/top-level/python-packages.nix

+3 −1
Original line number Diff line number Diff line
@@ -11936,7 +11936,9 @@ self: super: with self; { 
    inherit (pkgs.llvmPackages_rocm) openmp;

  };



  torch-bin = callPackage ../development/python-modules/torch/bin.nix { };

  torch-bin = callPackage ../development/python-modules/torch/bin.nix {

    openai-triton = self.openai-triton-bin;

  };



  torchWithCuda = self.torch.override {

    magma = pkgs.magma-cuda;