python312Packages.tokenizers: 0.20.0 -> 0.20.1 (#347707) (891b615a) · Commits · nix / nixpkgs

pkgs/development/python-modules/tokenizers/Cargo.lock

+66 −54

Original line number	Diff line number	Diff line
		@@ -62,9 +62,9 @@ dependencies = [

		[[package]]
		name = "autocfg"
		version = "1.3.0"
		version = "1.4.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
		checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"

		[[package]]
		name = "base64"
		@@ -92,9 +92,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"

		[[package]]
		name = "cc"
		version = "1.1.8"
		version = "1.1.22"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
		checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0"
		dependencies = [
		"shlex",
		]

		[[package]]
		name = "cfg-if"
		@@ -183,18 +186,18 @@ dependencies = [

		[[package]]
		name = "derive_builder"
		version = "0.20.0"
		version = "0.20.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
		checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b"
		dependencies = [
		"derive_builder_macro",
		]

		[[package]]
		name = "derive_builder_core"
		version = "0.20.0"
		version = "0.20.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
		checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38"
		dependencies = [
		"darling",
		"proc-macro2",
		@@ -204,9 +207,9 @@ dependencies = [

		[[package]]
		name = "derive_builder_macro"
		version = "0.20.0"
		version = "0.20.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
		checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc"
		dependencies = [
		"derive_builder_core",
		"syn",
		@@ -268,9 +271,9 @@ dependencies = [

		[[package]]
		name = "fastrand"
		version = "2.1.0"
		version = "2.1.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
		checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"

		[[package]]
		name = "fnv"
		@@ -373,9 +376,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"

		[[package]]
		name = "libc"
		version = "0.2.155"
		version = "0.2.159"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
		checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"

		[[package]]
		name = "linux-raw-sys"
		@@ -540,9 +543,12 @@ dependencies = [

		[[package]]
		name = "once_cell"
		version = "1.19.0"
		version = "1.20.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
		checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1"
		dependencies = [
		"portable-atomic",
		]

		[[package]]
		name = "onig"
		@@ -597,15 +603,15 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"

		[[package]]
		name = "pkg-config"
		version = "0.3.30"
		version = "0.3.31"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
		checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"

		[[package]]
		name = "portable-atomic"
		version = "1.7.0"
		version = "1.9.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
		checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"

		[[package]]
		name = "ppv-lite86"
		@@ -690,9 +696,9 @@ dependencies = [

		[[package]]
		name = "quote"
		version = "1.0.36"
		version = "1.0.37"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
		checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
		dependencies = [
		"proc-macro2",
		]
		@@ -766,18 +772,18 @@ dependencies = [

		[[package]]
		name = "redox_syscall"
		version = "0.5.3"
		version = "0.5.7"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
		checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
		dependencies = [
		"bitflags 2.6.0",
		]

		[[package]]
		name = "regex"
		version = "1.10.6"
		version = "1.11.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
		checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
		dependencies = [
		"aho-corasick",
		"memchr",
		@@ -787,9 +793,9 @@ dependencies = [

		[[package]]
		name = "regex-automata"
		version = "0.4.7"
		version = "0.4.8"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
		checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
		dependencies = [
		"aho-corasick",
		"memchr",
		@@ -798,9 +804,9 @@ dependencies = [

		[[package]]
		name = "regex-syntax"
		version = "0.8.4"
		version = "0.8.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
		checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"

		[[package]]
		name = "rustc-hash"
		@@ -810,9 +816,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"

		[[package]]
		name = "rustix"
		version = "0.38.34"
		version = "0.38.37"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
		checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
		dependencies = [
		"bitflags 2.6.0",
		"errno",
		@@ -835,18 +841,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

		[[package]]
		name = "serde"
		version = "1.0.205"
		version = "1.0.210"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
		checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
		dependencies = [
		"serde_derive",
		]

		[[package]]
		name = "serde_derive"
		version = "1.0.205"
		version = "1.0.210"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
		checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -855,9 +861,9 @@ dependencies = [

		[[package]]
		name = "serde_json"
		version = "1.0.122"
		version = "1.0.128"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
		checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
		dependencies = [
		"itoa",
		"memchr",
		@@ -865,6 +871,12 @@ dependencies = [
		"serde",
		]

		[[package]]
		name = "shlex"
		version = "1.3.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"

		[[package]]
		name = "smallvec"
		version = "1.13.2"
		@@ -891,9 +903,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"

		[[package]]
		name = "syn"
		version = "2.0.72"
		version = "2.0.79"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
		checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -908,9 +920,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"

		[[package]]
		name = "tempfile"
		version = "3.12.0"
		version = "3.13.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
		checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
		dependencies = [
		"cfg-if",
		"fastrand",
		@@ -921,18 +933,18 @@ dependencies = [

		[[package]]
		name = "thiserror"
		version = "1.0.63"
		version = "1.0.64"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
		checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
		dependencies = [
		"thiserror-impl",
		]

		[[package]]
		name = "thiserror-impl"
		version = "1.0.63"
		version = "1.0.64"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
		checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -941,7 +953,7 @@ dependencies = [

		[[package]]
		name = "tokenizers"
		version = "0.20.0-rc1"
		version = "0.20.0-dev.0"
		dependencies = [
		"aho-corasick",
		"derive_builder",
		@@ -971,7 +983,7 @@ dependencies = [

		[[package]]
		name = "tokenizers-python"
		version = "0.20.0-rc1"
		version = "0.20.0-dev.0"
		dependencies = [
		"env_logger",
		"itertools 0.12.1",
		@@ -988,9 +1000,9 @@ dependencies = [

		[[package]]
		name = "unicode-ident"
		version = "1.0.12"
		version = "1.0.13"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
		checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"

		[[package]]
		name = "unicode-normalization-alignments"
		@@ -1003,15 +1015,15 @@ dependencies = [

		[[package]]
		name = "unicode-segmentation"
		version = "1.11.0"
		version = "1.12.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
		checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"

		[[package]]
		name = "unicode-width"
		version = "0.1.13"
		version = "0.1.14"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
		checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"

		[[package]]
		name = "unicode_categories"

pkgs/development/python-modules/tokenizers/default.nix

+4 −3

Original line number	Diff line number	Diff line
		@@ -52,7 +52,7 @@ let
		};
		"big.txt" = fetchurl {
		url = "https://norvig.com/big.txt";
		sha256 = "sha256-+gZsfUDw8gGsQUTmUqpiQw5YprOAXscGUPZ42lgE6Hs=";
		hash = "sha256-+gZsfUDw8gGsQUTmUqpiQw5YprOAXscGUPZ42lgE6Hs=";
		};
		"bert-wiki.json" = fetchurl {
		url = "https://s3.amazonaws.com/models.huggingface.co/bert/anthony/doc-pipeline/tokenizer.json";
		@@ -74,14 +74,14 @@ let
		in
		buildPythonPackage rec {
		pname = "tokenizers";
		version = "0.20.0";
		version = "0.20.1";
		pyproject = true;

		src = fetchFromGitHub {
		owner = "huggingface";
		repo = "tokenizers";
		rev = "refs/tags/v${version}";
		hash = "sha256-uuSHsdyx77YQjf1aiz7EJ/X+6RaOgfmjGqHSlMaCWDI=";
		hash = "sha256-QTe1QdmJHSoosNG9cCJS7uQNdoMwgL+CJHQQUX5VtSY=";
		};

		cargoDeps = rustPlatform.importCargoLock { lockFile = ./Cargo.lock; };
		@@ -149,6 +149,7 @@ buildPythonPackage rec {
		meta = {
		description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";
		homepage = "https://github.com/huggingface/tokenizers";
		changelog = "https://github.com/huggingface/tokenizers/releases/tag/v${version}";
		license = lib.licenses.asl20;
		maintainers = with lib.maintainers; [ GaetanLepage ];
		platforms = lib.platforms.unix;

Admin message