Unverified Commit 0b7f9782 authored by Aleksana's avatar Aleksana Committed by GitHub
Browse files

python3Packages.sentencepiece: fix segfault on import for darwin (#470582)

parents c93bf642 9b2a5d79

pkgs/by-name/se/sentencepiece/package.nix

+6 −3
Original line number Diff line number Diff line
@@ -36,14 +36,17 @@ stdenv.mkDerivation rec { 
      --replace '\$'{prefix}/'$'{CMAKE_INSTALL_INCLUDEDIR} '$'{CMAKE_INSTALL_FULL_INCLUDEDIR}

  '';



  # On Darwin, non-static build segfaults on python module import.

  # See: https://github.com/NixOS/nixpkgs/issues/466092

  cmakeFlags = lib.optionals stdenv.hostPlatform.isDarwin [

    "-DSPM_ENABLE_SHARED=OFF"

  ];



  meta = {

    homepage = "https://github.com/google/sentencepiece";

    description = "Unsupervised text tokenizer for Neural Network-based text generation";

    license = lib.licenses.asl20;

    platforms = lib.platforms.unix;

    maintainers = with lib.maintainers; [ pashashocky ];

    # sentencepiece 0.2.1 segfaults on darwin when instantiated

    # See https://github.com/NixOS/nixpkgs/issues/466092

    badPlatforms = [ lib.systems.inspect.patterns.isDarwin ];

  };

}

pkgs/development/python-modules/gguf/default.nix

+1 −6
Original line number Diff line number Diff line 
{

  lib,

  stdenv,

  buildPythonPackage,

  fetchFromGitHub,
@@ -38,12 +37,8 @@ buildPythonPackage rec { 
    numpy

    pyside6

    pyyaml

    tqdm

  ]

  # Sentencepiece is optional and its inclusion crashes darwin

  # See https://github.com/NixOS/nixpkgs/issues/466092

  ++ lib.optionals stdenv.hostPlatform.isLinux [

    sentencepiece

    tqdm

  ];



  nativeCheckInputs = [ pytestCheckHook ];

pkgs/development/python-modules/mistral-common/default.nix

+4 −20
Original line number Diff line number Diff line 
{

  lib,

  stdenv,

  buildPythonPackage,

  fetchFromGitHub,
@@ -15,6 +14,7 @@ 
  pydantic,

  pydantic-extra-types,

  requests,

  sentencepiece,

  tiktoken,

  typing-extensions,
@@ -26,7 +26,6 @@ 
  pycountry,

  pydantic-settings,

  pytestCheckHook,

  sentencepiece,

  soundfile,

  soxr,

  uvicorn,
@@ -54,6 +53,7 @@ buildPythonPackage rec { 
    pydantic

    pydantic-extra-types

    requests

    sentencepiece

    tiktoken

    typing-extensions

  ];
@@ -62,8 +62,7 @@ buildPythonPackage rec { 
    opencv = [

      opencv-python-headless

    ];

    # Broken on Darwin. See https://github.com/NixOS/nixpkgs/issues/466092

    sentencepiece = lib.optionals (!stdenv.hostPlatform.isDarwin) [

    sentencepiece = [

      sentencepiece

    ];

    soundfile = [
@@ -98,8 +97,7 @@ buildPythonPackage rec { 
    soundfile

    soxr

    uvicorn

  ]

  ++ lib.concatAttrValues optional-dependencies;

  ];



  disabledTests = [

    # Require internet
@@ -114,20 +112,6 @@ buildPythonPackage rec { 
    "test_openai_chat_fields"

  ];



  # Requires sentencepiece which segfaults when initialized on Darwin

  # See https://github.com/NixOS/nixpkgs/issues/466092

  disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [

    "tests/experimental/test_app.py"

    "tests/experimental/test_tools.py"

    "tests/test_fim_tokenizer.py"

    "tests/test_integration_samples.py"

    "tests/test_mistral_tokenizer.py"

    "tests/test_tokenize_v1.py"

    "tests/test_tokenize_v2.py"

    "tests/test_tokenize_v3.py"

    "tests/test_tokenizer_v7.py"

  ];



  meta = {

    description = "Tools to help you work with Mistral models";

    homepage = "https://github.com/mistralai/mistral-common";

pkgs/development/python-modules/pytorch-tokenizers/default.nix

+0 −5
Original line number Diff line number Diff line
@@ -90,10 +90,5 @@ buildPythonPackage rec { 
    homepage = "https://github.com/meta-pytorch/tokenizers";

    license = lib.licenses.bsd3;

    maintainers = with lib.maintainers; [ GaetanLepage ];

    badPlatforms = [

      # sentencepiece 0.21.0 segfaults when initialized on Darwin

      # See https://github.com/NixOS/nixpkgs/issues/466092

      lib.systems.inspect.patterns.isDarwin

    ];

  };

}

pkgs/development/python-modules/sentencepiece/default.nix

+2 −0
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ buildPythonPackage rec { 


  sourceRoot = "${src.name}/python";



  pythonImportsCheck = [ "sentencepiece" ];



  # sentencepiece installs 'bin' output.

  meta = removeAttrs sentencepiece.meta [ "outputsToInstall" ];

}