Unverified Commit 5b87022c authored by OTABI Tomoya's avatar OTABI Tomoya Committed by GitHub
Browse files

Merge pull request #235226 from natsukium/catboost/update 

python3Packages.catboost: 1.0.5 -> 1.2.2; build with cmake
parents 6b8fd9de 0b93e562

pkgs/development/libraries/catboost/default.nix

0 → 100644
+113 −0
Original line number Diff line number Diff line 
{ lib

, config

, stdenv

, fetchFromGitHub

, cmake

, libiconv

, llvmPackages

, ninja

, openssl

, python3Packages

, ragel

, yasm

, zlib

, cudaSupport ? config.cudaSupport

, cudaPackages ? {}

, pythonSupport ? false

}:



stdenv.mkDerivation (finalAttrs: {

  pname = "catboost";

  version = "1.2.2";



  src = fetchFromGitHub {

    owner = "catboost";

    repo = "catboost";

    rev = "refs/tags/v${finalAttrs.version}";

    hash = "sha256-A1zCIqPOW21dHKBQHRtS+/sstZ2o6F8k71lmJFGn0+g=";

  };



  patches = [

    ./remove-conan.patch

  ];



  postPatch = ''

    substituteInPlace cmake/common.cmake \

      --replace  "\''${RAGEL_BIN}" "${ragel}/bin/ragel" \

      --replace "\''${YASM_BIN}" "${yasm}/bin/yasm"



    shopt -s globstar

    for cmakelists in **/CMakeLists.*; do

      sed -i "s/OpenSSL::OpenSSL/OpenSSL::SSL/g" $cmakelists

      ${lib.optionalString (lib.versionOlder cudaPackages.cudaVersion "11.8") ''

        sed -i 's/-gencode=arch=compute_89,code=sm_89//g' $cmakelists

        sed -i 's/-gencode=arch=compute_90,code=sm_90//g' $cmakelists

      ''}

    done

  '';



  outputs = [ "out" "dev" ];



  nativeBuildInputs = [

    cmake

    llvmPackages.bintools

    ninja

    (python3Packages.python.withPackages (ps: with ps; [ six ]))

    ragel

    yasm

  ] ++ lib.optionals cudaSupport (with cudaPackages; [

    cuda_nvcc

  ]);



  buildInputs = [

    openssl

    zlib

  ] ++ lib.optionals stdenv.isDarwin [

    libiconv

  ] ++ lib.optionals cudaSupport (with cudaPackages; [

    cuda_cudart

    cuda_cccl

    libcublas

  ]);



  env = {

    CUDAHOSTCXX = lib.optionalString cudaSupport "${stdenv.cc}/bin/cc";

    NIX_CFLAGS_LINK = lib.optionalString stdenv.isLinux "-fuse-ld=lld";

    NIX_LDFLAGS = "-lc -lm";

  };



  cmakeFlags = [

    "-DCMAKE_BINARY_DIR=$out"

    "-DCMAKE_POSITION_INDEPENDENT_CODE=on"

    "-DCATBOOST_COMPONENTS=app;libs${lib.optionalString pythonSupport ";python-package"}"

  ] ++ lib.optionals cudaSupport [

    "-DHAVE_CUDA=on"

  ];



  installPhase = ''

    runHook preInstall



    mkdir $dev

    cp -r catboost $dev

    install -Dm555 catboost/app/catboost -t $out/bin

    install -Dm444 catboost/libs/model_interface/static/lib/libmodel_interface-static-lib.a -t $out/lib

    install -Dm444 catboost/libs/model_interface/libcatboostmodel${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib

    install -Dm444 catboost/libs/train_interface/libcatboost${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib



    runHook postInstall

  '';



  meta = with lib; {

    description = "High-performance library for gradient boosting on decision trees";

    longDescription = ''

      A fast, scalable, high performance Gradient Boosting on Decision Trees

      library, used for ranking, classification, regression and other machine

      learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU.

    '';

    license = licenses.asl20;

    platforms = platforms.unix;

    homepage = "https://catboost.ai";

    maintainers = with maintainers; [ PlushBeaver natsukium ];

    mainProgram = "catboost";

  };

})

pkgs/development/libraries/catboost/remove-conan.patch

0 → 100644
+34 −0
Original line number Diff line number Diff line 
diff --git a/CMakeLists.txt b/CMakeLists.txt

index becd2ad03c..7e3c8c99b1 100644

--- a/CMakeLists.txt

+++ b/CMakeLists.txt

@@ -27,7 +27,6 @@ cmake_policy(SET CMP0104 OLD)

 

 include(cmake/archive.cmake)

 include(cmake/common.cmake)

-include(cmake/conan.cmake)

 include(cmake/cuda.cmake)

 include(cmake/cython.cmake)

 include(cmake/fbs.cmake)

@@ -37,21 +36,6 @@ include(cmake/recursive_library.cmake)

 include(cmake/swig.cmake)

 include(cmake/global_vars.cmake)

 

-if (CMAKE_CROSSCOMPILING)

-  include(${CMAKE_BINARY_DIR}/conan_paths.cmake)

-else()

-  conan_cmake_autodetect(settings)

-  conan_cmake_install(

-    PATH_OR_REFERENCE ${CMAKE_SOURCE_DIR}

-    INSTALL_FOLDER ${CMAKE_BINARY_DIR}

-    BUILD missing

-    REMOTE conancenter

-    SETTINGS ${settings}

-      ENV "CONAN_CMAKE_GENERATOR=${CMAKE_GENERATOR}"

-      CONF "tools.cmake.cmaketoolchain:generator=${CMAKE_GENERATOR}"

-  )

-endif()

-

 if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)

   include(CMakeLists.linux-x86_64.txt)

 elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND HAVE_CUDA)

pkgs/development/python-modules/catboost/default.nix

+41 −55
Original line number Diff line number Diff line 
{ buildPythonPackage, fetchFromGitHub, lib, pythonOlder

, clang_12, python

, graphviz, matplotlib, numpy, pandas, plotly, scipy, six

, withCuda ? false, cudatoolkit }:



buildPythonPackage rec {

  pname = "catboost";

  # nixpkgs-update: no auto update

  version = "1.0.5";



  disabled = pythonOlder "3.4";



  src = fetchFromGitHub {

    owner = "catboost";

    repo = "catboost";

    rev = "refs/tags/v${version}";

    hash = "sha256-ILemeZUBI9jPb9G6F7QX/T1HaVhQ+g6y7YmsT6DFCJk";

  };

{ lib

, buildPythonPackage

, catboost

, python

, graphviz

, matplotlib

, numpy

, pandas

, plotly

, scipy

, setuptools

, six

, wheel

}:



buildPythonPackage {

  inherit (catboost) pname version src meta;

  format = "pyproject";



  sourceRoot = "source/catboost/python-package";



  nativeBuildInputs = [

    setuptools

    wheel

  ];



  nativeBuildInputs = [ clang_12 ];

  propagatedBuildInputs = [

    graphviz

    matplotlib

    numpy

    pandas

    plotly

    scipy

    six

  ];



  propagatedBuildInputs = [ graphviz matplotlib numpy pandas scipy plotly six ]

    ++ lib.optionals withCuda [ cudatoolkit ];

  buildPhase = ''

    runHook preBuild



  patches = [

    ./nix-support.patch

  ];

    # these arguments must set after bdist_wheel

    ${python.pythonForBuild.interpreter} setup.py bdist_wheel --no-widget --prebuilt-extensions-build-root-dir=${lib.getDev catboost}



  postPatch = ''

    # substituteInPlace is too slow for these large files, and the target has lots of numbers in it that change often.

    sed -e 's|\$(YMAKE_PYTHON3-.*)/python3|${python.interpreter}|' -i make/*.makefile

    runHook postBuild

  '';



  preBuild = ''

    cd catboost/python-package

  '';

  setupPyBuildFlags = [ "--with-ymake=no" ];

  CUDA_ROOT = lib.optional withCuda cudatoolkit;

  enableParallelBuilding = true;

  # setup a test is difficult

  doCheck = false;



  # Tests use custom "ya" tool, not yet supported.

  dontUseSetuptoolsCheck = true;

  pythonImportsCheck = [ "catboost" ];



  passthru = {

    # Do not update to catboost 1.1.x because the patch doesn't apply cleanly

    skipBulkUpdate = true;

  };



  meta = with lib; {

    description = "High-performance library for gradient boosting on decision trees.";

    longDescription = ''

      A fast, scalable, high performance Gradient Boosting on Decision Trees

      library, used for ranking, classification, regression and other machine

      learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU.

    '';

    license = licenses.asl20;

    platforms = [ "x86_64-linux" ];

    homepage = "https://catboost.ai";

    maintainers = with maintainers; [ PlushBeaver ];

    # _catboost.pyx.cpp:226822:19: error: use of undeclared identifier '_PyGen_Send'

    broken = withCuda;

  };

}

pkgs/development/python-modules/catboost/nix-support.patch

deleted100644 → 0
+0 −173
Original line number Diff line number Diff line 
diff --git a/catboost/python-package/setup.py b/catboost/python-package/setup.py

index fe9251a21f..86b880c5d0 100644

--- a/catboost/python-package/setup.py

+++ b/catboost/python-package/setup.py

@@ -80,7 +80,7 @@ class Helper(object):

         self.with_cuda = os.environ.get('CUDA_PATH') or os.environ.get('CUDA_ROOT') or None

         self.os_sdk = 'local'

         self.with_ymake = True

-        self.parallel = None

+        self.parallel = os.environ.get('NIX_BUILD_CORES') or None

 

     def finalize_options(self):

         if os.path.exists(str(self.with_cuda)):

@@ -222,11 +222,12 @@ class build_ext(_build_ext):

 

     def build_with_make(self, topsrc_dir, build_dir, catboost_ext, put_dir, verbose, dry_run):

         logging.info('Buildling {} with gnu make'.format(catboost_ext))

-        makefile = 'python{}.{}CLANG11-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '')

+        makefile = 'python{}.{}CLANG12-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '')

         make_cmd = [

             'make', '-f', '../../make/' + makefile,

-            'CC=clang-11',

-            'CXX=clang++-11',

+            'CC=clang',

+            'CXX=clang++',

+            'PYTHON=python{}'.format(python_version()[0]),

             'BUILD_ROOT=' + build_dir,

             'SOURCE_ROOT=' + topsrc_dir,

         ]

diff --git a/make/python2.CLANG12-LINUX-X86_64.makefile b/make/python2.CLANG12-LINUX-X86_64.makefile

index b49a36fb3f..33996af995 100644

--- a/make/python2.CLANG12-LINUX-X86_64.makefile

+++ b/make/python2.CLANG12-LINUX-X86_64.makefile

@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)

 SOURCE_ROOT = $(shell pwd)

 PYTHON = $(shell which python)

 

-ifneq ($(MAKECMDGOALS),help)

-define _CC_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)

-$(info _CC_VERSION = '$(_CC_VERSION)')

-

-ifneq '$(_CC_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

-

-ifneq ($(MAKECMDGOALS),help)

-define _CXX_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)

-$(info _CXX_VERSION = '$(_CXX_VERSION)')

-

-ifneq '$(_CXX_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

 

 

 all\

diff --git a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile

index 82935b297e..093cc86532 100644

--- a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile

+++ b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile

@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)

 SOURCE_ROOT = $(shell pwd)

 PYTHON = $(shell which python)

 

-ifneq ($(MAKECMDGOALS),help)

-define _CC_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)

-$(info _CC_VERSION = '$(_CC_VERSION)')

-

-ifneq '$(_CC_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

-

-ifneq ($(MAKECMDGOALS),help)

-define _CXX_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)

-$(info _CXX_VERSION = '$(_CXX_VERSION)')

-

-ifneq '$(_CXX_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

 

 

 all\

diff --git a/make/python3.CLANG12-LINUX-X86_64.makefile b/make/python3.CLANG12-LINUX-X86_64.makefile

index 1c5d646ae4..6c091fbe17 100644

--- a/make/python3.CLANG12-LINUX-X86_64.makefile

+++ b/make/python3.CLANG12-LINUX-X86_64.makefile

@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)

 SOURCE_ROOT = $(shell pwd)

 PYTHON = $(shell which python)

 

-ifneq ($(MAKECMDGOALS),help)

-define _CC_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)

-$(info _CC_VERSION = '$(_CC_VERSION)')

-

-ifneq '$(_CC_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

-

-ifneq ($(MAKECMDGOALS),help)

-define _CXX_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)

-$(info _CXX_VERSION = '$(_CXX_VERSION)')

-

-ifneq '$(_CXX_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

 

 

 all\

diff --git a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile

index fcdb75a719..4e1dbc3cd7 100644

--- a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile

+++ b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile

@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd)

 SOURCE_ROOT = $(shell pwd)

 PYTHON = $(shell which python)

 

-ifneq ($(MAKECMDGOALS),help)

-define _CC_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -)

-$(info _CC_VERSION = '$(_CC_VERSION)')

-

-ifneq '$(_CC_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

-

-ifneq ($(MAKECMDGOALS),help)

-define _CXX_TEST

-__clang_major__ __clang_minor__

-endef

-

-_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -)

-$(info _CXX_VERSION = '$(_CXX_VERSION)')

-

-ifneq '$(_CXX_VERSION)' '12 0'

-    $(error clang 12.0 is required)

-endif

-endif

 

 

 all\

pkgs/top-level/all-packages.nix

+6 −0
Original line number Diff line number Diff line
@@ -20755,6 +20755,12 @@ with pkgs; 














  captive-browser = callPackage ../applications/networking/browsers/captive-browser { };



























  catboost = callPackage ../development/libraries/catboost {













    # catboost requires clang 12+ for build













    # after bumping the default version of llvm, check for compatibility with the cuda backend and pin it.













    inherit (llvmPackages_12) stdenv;













  };



























  ndn-cxx = callPackage ../development/libraries/ndn-cxx { };





























  ndn-tools = callPackage ../tools/networking/ndn-tools { };