Merge pull request #326691 from natsukium/tiktoken/update (7bbfd756) · Commits · nix / nixpkgs

pkgs/development/python-modules/litellm/default.nix

+13 −12

Original line number	Diff line number	Diff line
		@@ -7,6 +7,7 @@
		backoff,
		buildPythonPackage,
		click,
		cryptography,
		fastapi,
		fastapi-sso,
		fetchFromGitHub,
		@@ -14,11 +15,14 @@
		gunicorn,
		importlib-metadata,
		jinja2,
		jsonschema,
		openai,
		orjson,
		poetry-core,
		prisma,
		pydantic,
		pyjwt,
		pynacl,
		python-dotenv,
		python-multipart,
		pythonOlder,
		@@ -26,7 +30,6 @@
		requests,
		resend,
		rq,
		streamlit,
		tiktoken,
		tokenizers,
		uvicorn,
		@@ -34,7 +37,7 @@

		buildPythonPackage rec {
		pname = "litellm";
		version = "1.40.16";
		version = "1.41.19";
		pyproject = true;

		disabled = pythonOlder "3.8";
		@@ -43,13 +46,9 @@ buildPythonPackage rec {
		owner = "BerriAI";
		repo = "litellm";
		rev = "refs/tags/v${version}";
		hash = "sha256-CK/b0PVBOzfhnTk+iu/buu7BIjAGdz3aXYAGgB4s/pw=";
		hash = "sha256-IM91EHrfgYdsG+/9Ro/SV0Dko1dc7KOLDU76ar7/lew=";
		};

		postPatch = ''
		rm -rf dist
		'';

		build-system = [ poetry-core ];

		dependencies = [
		@@ -57,9 +56,11 @@ buildPythonPackage rec {
		click
		importlib-metadata
		jinja2
		jsonschema
		openai
		requests
		pydantic
		python-dotenv
		requests
		tiktoken
		tokenizers
		];
		@@ -68,6 +69,7 @@ buildPythonPackage rec {
		proxy = [
		apscheduler
		backoff
		cryptography
		fastapi
		fastapi-sso
		gunicorn
		@@ -83,15 +85,14 @@ buildPythonPackage rec {
		azure-keyvault-secrets
		google-cloud-kms
		prisma
		pynacl
		resend
		streamlit
		];
		};

		# the import check phase fails trying to do a network request to openai
		# pythonImportsCheck = [ "litellm" ];
		pythonImportsCheck = [ "litellm" ];

		# no tests
		# access network
		doCheck = false;

		meta = with lib; {

pkgs/development/python-modules/tiktoken/Cargo.lock

+100 −89

Original line number	Diff line number	Diff line
		@@ -4,18 +4,18 @@ version = 3

		[[package]]
		name = "aho-corasick"
		version = "1.0.5"
		version = "1.1.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783"
		checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
		dependencies = [
		"memchr",
		]

		[[package]]
		name = "autocfg"
		version = "1.1.0"
		version = "1.3.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
		checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"

		[[package]]
		name = "bit-set"
		@@ -34,15 +34,15 @@ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"

		[[package]]
		name = "bitflags"
		version = "1.3.2"
		version = "2.6.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
		checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"

		[[package]]
		name = "bstr"
		version = "1.6.2"
		version = "1.9.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a"
		checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
		dependencies = [
		"memchr",
		"regex-automata",
		@@ -65,23 +65,29 @@ dependencies = [
		"regex",
		]

		[[package]]
		name = "heck"
		version = "0.4.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"

		[[package]]
		name = "indoc"
		version = "1.0.9"
		version = "2.0.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306"
		checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"

		[[package]]
		name = "libc"
		version = "0.2.148"
		version = "0.2.155"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
		checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"

		[[package]]
		name = "lock_api"
		version = "0.4.10"
		version = "0.4.12"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
		checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
		dependencies = [
		"autocfg",
		"scopeguard",
		@@ -89,30 +95,30 @@ dependencies = [

		[[package]]
		name = "memchr"
		version = "2.6.3"
		version = "2.7.4"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
		checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"

		[[package]]
		name = "memoffset"
		version = "0.9.0"
		version = "0.9.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
		checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
		dependencies = [
		"autocfg",
		]

		[[package]]
		name = "once_cell"
		version = "1.18.0"
		version = "1.19.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
		checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"

		[[package]]
		name = "parking_lot"
		version = "0.12.1"
		version = "0.12.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
		checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
		dependencies = [
		"lock_api",
		"parking_lot_core",
		@@ -120,9 +126,9 @@ dependencies = [

		[[package]]
		name = "parking_lot_core"
		version = "0.9.8"
		version = "0.9.10"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
		checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
		dependencies = [
		"cfg-if",
		"libc",
		@@ -131,26 +137,33 @@ dependencies = [
		"windows-targets",
		]

		[[package]]
		name = "portable-atomic"
		version = "1.6.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"

		[[package]]
		name = "proc-macro2"
		version = "1.0.67"
		version = "1.0.86"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328"
		checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
		dependencies = [
		"unicode-ident",
		]

		[[package]]
		name = "pyo3"
		version = "0.19.2"
		version = "0.20.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38"
		checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233"
		dependencies = [
		"cfg-if",
		"indoc",
		"libc",
		"memoffset",
		"parking_lot",
		"portable-atomic",
		"pyo3-build-config",
		"pyo3-ffi",
		"pyo3-macros",
		@@ -159,9 +172,9 @@ dependencies = [

		[[package]]
		name = "pyo3-build-config"
		version = "0.19.2"
		version = "0.20.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5"
		checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7"
		dependencies = [
		"once_cell",
		"target-lexicon",
		@@ -169,9 +182,9 @@ dependencies = [

		[[package]]
		name = "pyo3-ffi"
		version = "0.19.2"
		version = "0.20.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9"
		checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa"
		dependencies = [
		"libc",
		"pyo3-build-config",
		@@ -179,50 +192,52 @@ dependencies = [

		[[package]]
		name = "pyo3-macros"
		version = "0.19.2"
		version = "0.20.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1"
		checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158"
		dependencies = [
		"proc-macro2",
		"pyo3-macros-backend",
		"quote",
		"syn 1.0.109",
		"syn",
		]

		[[package]]
		name = "pyo3-macros-backend"
		version = "0.19.2"
		version = "0.20.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536"
		checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185"
		dependencies = [
		"heck",
		"proc-macro2",
		"pyo3-build-config",
		"quote",
		"syn 1.0.109",
		"syn",
		]

		[[package]]
		name = "quote"
		version = "1.0.33"
		version = "1.0.36"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
		checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
		dependencies = [
		"proc-macro2",
		]

		[[package]]
		name = "redox_syscall"
		version = "0.3.5"
		version = "0.5.2"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
		checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd"
		dependencies = [
		"bitflags",
		]

		[[package]]
		name = "regex"
		version = "1.9.5"
		version = "1.10.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
		checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
		dependencies = [
		"aho-corasick",
		"memchr",
		@@ -232,9 +247,9 @@ dependencies = [

		[[package]]
		name = "regex-automata"
		version = "0.3.8"
		version = "0.4.7"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
		checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
		dependencies = [
		"aho-corasick",
		"memchr",
		@@ -243,9 +258,9 @@ dependencies = [

		[[package]]
		name = "regex-syntax"
		version = "0.7.5"
		version = "0.8.4"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
		checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"

		[[package]]
		name = "rustc-hash"
		@@ -261,46 +276,35 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

		[[package]]
		name = "serde"
		version = "1.0.188"
		version = "1.0.204"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
		checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
		dependencies = [
		"serde_derive",
		]

		[[package]]
		name = "serde_derive"
		version = "1.0.188"
		version = "1.0.204"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
		checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
		dependencies = [
		"proc-macro2",
		"quote",
		"syn 2.0.36",
		"syn",
		]

		[[package]]
		name = "smallvec"
		version = "1.11.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"

		[[package]]
		name = "syn"
		version = "1.0.109"
		version = "1.13.2"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
		dependencies = [
		"proc-macro2",
		"quote",
		"unicode-ident",
		]
		checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"

		[[package]]
		name = "syn"
		version = "2.0.36"
		version = "2.0.71"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "91e02e55d62894af2a08aca894c6577281f76769ba47c94d5756bec8ac6e7373"
		checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -309,13 +313,13 @@ dependencies = [

		[[package]]
		name = "target-lexicon"
		version = "0.12.11"
		version = "0.12.15"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a"
		checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2"

		[[package]]
		name = "tiktoken"
		version = "0.5.1"
		version = "0.7.0"
		dependencies = [
		"bstr",
		"fancy-regex",
		@@ -332,19 +336,20 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

		[[package]]
		name = "unindent"
		version = "0.1.11"
		version = "0.2.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c"
		checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"

		[[package]]
		name = "windows-targets"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
		checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
		dependencies = [
		"windows_aarch64_gnullvm",
		"windows_aarch64_msvc",
		"windows_i686_gnu",
		"windows_i686_gnullvm",
		"windows_i686_msvc",
		"windows_x86_64_gnu",
		"windows_x86_64_gnullvm",
		@@ -353,42 +358,48 @@ dependencies = [

		[[package]]
		name = "windows_aarch64_gnullvm"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
		checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"

		[[package]]
		name = "windows_aarch64_msvc"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
		checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"

		[[package]]
		name = "windows_i686_gnu"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"

		[[package]]
		name = "windows_i686_gnullvm"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
		checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"

		[[package]]
		name = "windows_i686_msvc"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
		checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"

		[[package]]
		name = "windows_x86_64_gnu"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
		checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"

		[[package]]
		name = "windows_x86_64_gnullvm"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
		checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"

		[[package]]
		name = "windows_x86_64_msvc"
		version = "0.48.5"
		version = "0.52.6"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
		checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

pkgs/development/python-modules/tiktoken/default.nix

+6 −8

Original line number	Diff line number	Diff line
		@@ -9,7 +9,6 @@
		rustc,
		setuptools,
		setuptools-rust,
		wheel,
		libiconv,
		requests,
		regex,
		@@ -17,10 +16,10 @@
		}:
		let
		pname = "tiktoken";
		version = "0.5.1";
		version = "0.7.0";
		src = fetchPypi {
		inherit pname version;
		hash = "sha256-J+dzVkIyAE9PgQ/R+FI2Zz7DpW7X8SBvye2GcOvtuXo=";
		hash = "sha256-EHcmbpScJOApH2w1BDPG8JcTZezisXOiO8O5+d7+9rY=";
		};
		postPatch = ''
		cp ${./Cargo.lock} Cargo.lock
		@@ -33,20 +32,19 @@ buildPythonPackage {
		src
		postPatch
		;
		format = "pyproject";
		pyproject = true;

		disabled = pythonOlder "3.8";

		nativeBuildInput = [
		build-system = [
		setuptools
		setuptools-rust
		wheel
		];

		cargoDeps = rustPlatform.fetchCargoTarball {
		inherit src postPatch;
		name = "${pname}-${version}";
		hash = "sha256-Q7XO+auj4tKDAGbqNn9pmJg8EJvooN2ie0lWwZVrld4=";
		hash = "sha256-i0AQUu9ERDWBw0kjTTTyn4VHMig/k2/7wX2884MCGx8=";
		};

		nativeBuildInputs = [
		@@ -58,7 +56,7 @@ buildPythonPackage {

		buildInputs = lib.optionals stdenv.isDarwin [ libiconv ];

		propagatedBuildInputs = [
		dependencies = [
		requests
		regex
		blobfile

Admin message