python312Packages.tokenizers: 0.20.1 -> 0.20.2 (#353766) (3fc97935) · Commits · nix / nixpkgs

pkgs/development/python-modules/sagemaker/default.nix

+1 −0

Original line number	Diff line number	Diff line
		@@ -53,6 +53,7 @@ buildPythonPackage rec {
		];

		pythonRelaxDeps = [
		"attrs"
		"boto3"
		"cloudpickle"
		"importlib-metadata"

pkgs/development/python-modules/tokenizers/Cargo.lock

+87 −114

Original line number	Diff line number	Diff line
		@@ -13,9 +13,9 @@ dependencies = [

		[[package]]
		name = "anstream"
		version = "0.6.15"
		version = "0.6.17"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
		checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338"
		dependencies = [
		"anstyle",
		"anstyle-parse",
		@@ -28,36 +28,36 @@ dependencies = [

		[[package]]
		name = "anstyle"
		version = "1.0.8"
		version = "1.0.10"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
		checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"

		[[package]]
		name = "anstyle-parse"
		version = "0.2.5"
		version = "0.2.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
		checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
		dependencies = [
		"utf8parse",
		]

		[[package]]
		name = "anstyle-query"
		version = "1.1.1"
		version = "1.1.2"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
		checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
		dependencies = [
		"windows-sys 0.52.0",
		"windows-sys 0.59.0",
		]

		[[package]]
		name = "anstyle-wincon"
		version = "3.0.4"
		version = "3.0.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
		checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125"
		dependencies = [
		"anstyle",
		"windows-sys 0.52.0",
		"windows-sys 0.59.0",
		]

		[[package]]
		@@ -92,9 +92,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"

		[[package]]
		name = "cc"
		version = "1.1.22"
		version = "1.1.34"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0"
		checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9"
		dependencies = [
		"shlex",
		]
		@@ -107,9 +107,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

		[[package]]
		name = "colorchoice"
		version = "1.0.2"
		version = "1.0.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
		checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"

		[[package]]
		name = "console"
		@@ -186,18 +186,18 @@ dependencies = [

		[[package]]
		name = "derive_builder"
		version = "0.20.1"
		version = "0.20.2"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b"
		checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
		dependencies = [
		"derive_builder_macro",
		]

		[[package]]
		name = "derive_builder_core"
		version = "0.20.1"
		version = "0.20.2"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38"
		checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
		dependencies = [
		"darling",
		"proc-macro2",
		@@ -207,9 +207,9 @@ dependencies = [

		[[package]]
		name = "derive_builder_macro"
		version = "0.20.1"
		version = "0.20.2"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc"
		checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
		dependencies = [
		"derive_builder_core",
		"syn",
		@@ -294,9 +294,9 @@ dependencies = [

		[[package]]
		name = "heck"
		version = "0.4.1"
		version = "0.5.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
		checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"

		[[package]]
		name = "humantime"
		@@ -376,9 +376,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"

		[[package]]
		name = "libc"
		version = "0.2.159"
		version = "0.2.161"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
		checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"

		[[package]]
		name = "linux-raw-sys"
		@@ -386,16 +386,6 @@ version = "0.4.14"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"

		[[package]]
		name = "lock_api"
		version = "0.4.12"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
		dependencies = [
		"autocfg",
		"scopeguard",
		]

		[[package]]
		name = "log"
		version = "0.4.22"
		@@ -483,6 +473,21 @@ dependencies = [
		"rawpointer",
		]

		[[package]]
		name = "ndarray"
		version = "0.16.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
		dependencies = [
		"matrixmultiply",
		"num-complex",
		"num-integer",
		"num-traits",
		"portable-atomic",
		"portable-atomic-util",
		"rawpointer",
		]

		[[package]]
		name = "nom"
		version = "7.1.3"
		@@ -528,12 +533,12 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"

		[[package]]
		name = "numpy"
		version = "0.21.0"
		version = "0.22.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "ec170733ca37175f5d75a5bea5911d6ff45d2cd52849ce98b685394e4f2f37f4"
		checksum = "edb929bc0da91a4d85ed6c0a84deaa53d411abfb387fc271124f91bf6b89f14e"
		dependencies = [
		"libc",
		"ndarray",
		"ndarray 0.16.1",
		"num-complex",
		"num-integer",
		"num-traits",
		@@ -543,12 +548,9 @@ dependencies = [

		[[package]]
		name = "once_cell"
		version = "1.20.1"
		version = "1.20.2"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1"
		dependencies = [
		"portable-atomic",
		]
		checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"

		[[package]]
		name = "onig"
		@@ -572,29 +574,6 @@ dependencies = [
		"pkg-config",
		]

		[[package]]
		name = "parking_lot"
		version = "0.12.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
		dependencies = [
		"lock_api",
		"parking_lot_core",
		]

		[[package]]
		name = "parking_lot_core"
		version = "0.9.10"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
		dependencies = [
		"cfg-if",
		"libc",
		"redox_syscall",
		"smallvec",
		"windows-targets",
		]

		[[package]]
		name = "paste"
		version = "1.0.15"
		@@ -613,6 +592,15 @@ version = "1.9.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"

		[[package]]
		name = "portable-atomic-util"
		version = "0.2.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "90a7d5beecc52a491b54d6dd05c7a45ba1801666a5baad9fdbfc6fef8d2d206c"
		dependencies = [
		"portable-atomic",
		]

		[[package]]
		name = "ppv-lite86"
		version = "0.2.20"
		@@ -624,24 +612,24 @@ dependencies = [

		[[package]]
		name = "proc-macro2"
		version = "1.0.86"
		version = "1.0.89"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
		checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
		dependencies = [
		"unicode-ident",
		]

		[[package]]
		name = "pyo3"
		version = "0.21.2"
		version = "0.22.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8"
		checksum = "3d922163ba1f79c04bc49073ba7b32fd5a8d3b76a87c955921234b8e77333c51"
		dependencies = [
		"cfg-if",
		"indoc",
		"libc",
		"memoffset",
		"parking_lot",
		"once_cell",
		"portable-atomic",
		"pyo3-build-config",
		"pyo3-ffi",
		@@ -651,9 +639,9 @@ dependencies = [

		[[package]]
		name = "pyo3-build-config"
		version = "0.21.2"
		version = "0.22.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50"
		checksum = "bc38c5feeb496c8321091edf3d63e9a6829eab4b863b4a6a65f26f3e9cc6b179"
		dependencies = [
		"once_cell",
		"target-lexicon",
		@@ -661,9 +649,9 @@ dependencies = [

		[[package]]
		name = "pyo3-ffi"
		version = "0.21.2"
		version = "0.22.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403"
		checksum = "94845622d88ae274d2729fcefc850e63d7a3ddff5e3ce11bd88486db9f1d357d"
		dependencies = [
		"libc",
		"pyo3-build-config",
		@@ -671,9 +659,9 @@ dependencies = [

		[[package]]
		name = "pyo3-macros"
		version = "0.21.2"
		version = "0.22.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c"
		checksum = "e655aad15e09b94ffdb3ce3d217acf652e26bbc37697ef012f5e5e348c716e5e"
		dependencies = [
		"proc-macro2",
		"pyo3-macros-backend",
		@@ -683,9 +671,9 @@ dependencies = [

		[[package]]
		name = "pyo3-macros-backend"
		version = "0.21.2"
		version = "0.22.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c"
		checksum = "ae1e3f09eecd94618f60a455a23def79f79eba4dc561a97324bf9ac8c6df30ce"
		dependencies = [
		"heck",
		"proc-macro2",
		@@ -770,20 +758,11 @@ dependencies = [
		"crossbeam-utils",
		]

		[[package]]
		name = "redox_syscall"
		version = "0.5.7"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
		dependencies = [
		"bitflags 2.6.0",
		]

		[[package]]
		name = "regex"
		version = "1.11.0"
		version = "1.11.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
		checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
		dependencies = [
		"aho-corasick",
		"memchr",
		@@ -816,9 +795,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"

		[[package]]
		name = "rustix"
		version = "0.38.37"
		version = "0.38.38"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
		checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a"
		dependencies = [
		"bitflags 2.6.0",
		"errno",
		@@ -833,26 +812,20 @@ version = "1.0.18"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"

		[[package]]
		name = "scopeguard"
		version = "1.2.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

		[[package]]
		name = "serde"
		version = "1.0.210"
		version = "1.0.214"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
		checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
		dependencies = [
		"serde_derive",
		]

		[[package]]
		name = "serde_derive"
		version = "1.0.210"
		version = "1.0.214"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
		checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -861,9 +834,9 @@ dependencies = [

		[[package]]
		name = "serde_json"
		version = "1.0.128"
		version = "1.0.132"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
		checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
		dependencies = [
		"itoa",
		"memchr",
		@@ -903,9 +876,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"

		[[package]]
		name = "syn"
		version = "2.0.79"
		version = "2.0.87"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
		checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -933,18 +906,18 @@ dependencies = [

		[[package]]
		name = "thiserror"
		version = "1.0.64"
		version = "1.0.68"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
		checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892"
		dependencies = [
		"thiserror-impl",
		]

		[[package]]
		name = "thiserror-impl"
		version = "1.0.64"
		version = "1.0.68"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
		checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -953,7 +926,7 @@ dependencies = [

		[[package]]
		name = "tokenizers"
		version = "0.20.0-dev.0"
		version = "0.20.2"
		dependencies = [
		"aho-corasick",
		"derive_builder",
		@@ -983,12 +956,12 @@ dependencies = [

		[[package]]
		name = "tokenizers-python"
		version = "0.20.0-dev.0"
		version = "0.20.2"
		dependencies = [
		"env_logger",
		"itertools 0.12.1",
		"libc",
		"ndarray",
		"ndarray 0.15.6",
		"numpy",
		"pyo3",
		"rayon",

pkgs/development/python-modules/tokenizers/default.nix

+2 −2

Original line number	Diff line number	Diff line
		@@ -74,14 +74,14 @@ let
		in
		buildPythonPackage rec {
		pname = "tokenizers";
		version = "0.20.1";
		version = "0.20.2";
		pyproject = true;

		src = fetchFromGitHub {
		owner = "huggingface";
		repo = "tokenizers";
		rev = "refs/tags/v${version}";
		hash = "sha256-QTe1QdmJHSoosNG9cCJS7uQNdoMwgL+CJHQQUX5VtSY=";
		hash = "sha256-6uUlDLL1dMg5/6/z8AuB7ibqSJ7IUVexHDTkLx35wFY=";
		};

		cargoDeps = rustPlatform.importCargoLock { lockFile = ./Cargo.lock; };