Commit 5e766a1a authored by Gaetan Lepage's avatar Gaetan Lepage
Browse files

python311Packages.tokenizers: 0.15.0 -> 0.19.1 

Diff: https://github.com/huggingface/tokenizers/compare/v0.15.0...v0.19.1
parent 92d295f5

pkgs/development/python-modules/tokenizers/Cargo.lock

+224 −332

File changed.

Preview size limit exceeded, changes collapsed.

pkgs/development/python-modules/tokenizers/default.nix

+14 −6
Original line number Diff line number Diff line
@@ -63,16 +63,16 @@ let 
in

buildPythonPackage rec {

  pname = "tokenizers";

  version = "0.15.0";

  format = "pyproject";

  version = "0.19.1";

  pyproject = true;



  disabled = pythonOlder "3.7";



  src = fetchFromGitHub {

    owner = "huggingface";

    repo = pname;

    rev = "v${version}";

    hash = "sha256-+yfX12eKtgZV1OQvPOlMVTONbpFuigHcl4SjoCIZkSk=";

    repo = "tokenizers";

    rev = "refs/tags/v${version}";

    hash = "sha256-sKEAt46cdme821tzz9WSKnQb3hPmFJ4zvHgBNRxjEuk=";

  };



  cargoDeps = rustPlatform.importCargoLock {
@@ -97,7 +97,13 @@ buildPythonPackage rec { 
    Security

  ];



  propagatedBuildInputs = [

  # Cargo.lock is outdated

  # TODO: remove at next release

  preConfigure = ''

    cargo update --offline

  '';



  dependencies = [

    numpy

  ];
@@ -123,6 +129,8 @@ buildPythonPackage rec { 


  disabledTests = [

    # Downloads data using the datasets module

    "test_encode_special_tokens"

    "test_splitting"

    "TestTrainFromIterators"

    # Those tests require more data

    "test_from_pretrained"