vimPluginsUpdater: fix treesitter updates (252412c2) · Commits · nix / nixpkgs

maintainers/scripts/pluginupdate.py

+2 −4

Original line number	Diff line number	Diff line
		@@ -327,7 +327,6 @@ def run_nix_expr(expr, nixpkgs: str):
		:param expr nix expression to fetch current plugins
		:param nixpkgs Path towards a nixpkgs checkout
		'''
		# local_pkgs = str(Path(__file__).parent.parent.parent)
		with CleanEnvironment(nixpkgs) as nix_path:
		cmd = [
		"nix",
		@@ -341,8 +340,8 @@ def run_nix_expr(expr, nixpkgs: str):
		"--nix-path",
		nix_path,
		]
		log.debug("Running command %s", " ".join(cmd))
		out = subprocess.check_output(cmd)
		log.debug("Running command: %s", " ".join(cmd))
		out = subprocess.check_output(cmd, timeout=90)
		data = json.loads(out)
		return data

		@@ -572,7 +571,6 @@ class CleanEnvironment(object):
		self.empty_config = NamedTemporaryFile()
		self.empty_config.write(b"{}")
		self.empty_config.flush()
		# os.environ["NIXPKGS_CONFIG"] = self.empty_config.name
		return f"localpkgs={self.local_pkgs}"

		def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:

pkgs/applications/editors/vim/plugins/nvim-treesitter/update.py

+47 −31

Original line number	Diff line number	Diff line
		@@ -2,34 +2,20 @@
		#!nix-shell update-shell.nix -i python

		import json
		import logging
		import subprocess
		from concurrent.futures import ThreadPoolExecutor
		from os import environ
		from os.path import dirname, join
		import os
		import sys
		from os.path import join

		configs = json.loads(
		subprocess.check_output(
		[
		"nvim",
		"--headless",
		"-u",
		"NONE",
		"+lua io.write(vim.json.encode(require('nvim-treesitter.parsers').get_parser_configs()))",
		"+quit!",
		]
		)
		)
		log = logging.getLogger("vim-updater")


		def generate_grammar(item):
		lang, lock = item
		cfg = configs.get(lang)
		if not cfg:
		return ""

		def generate_grammar(lang, rev, cfg):
		"""Generate grammar for a language"""
		info = cfg["install_info"]
		url = info["url"]
		rev = lock["revision"]

		generated = f""" {lang} = buildGrammar {{
		language = "{lang}";
		@@ -56,7 +42,24 @@ def generate_grammar(item):
		return generated


		def update_grammars(lockfile: str):
		def update_grammars(nvim_treesitter_dir: str):
		"""
		The lockfile contains just revisions so we start neovim to dump the
		grammar information in a better format
		"""
		# the lockfile
		cmd = [
		"nvim",
		"--headless",
		"-u",
		"NONE",
		"--cmd",
		f"set rtp^={nvim_treesitter_dir}",
		"+lua io.write(vim.json.encode(require('nvim-treesitter.parsers').get_parser_configs()))",
		"+quit!",
		]
		log.debug("Running command: %s", ' '.join(cmd))
		configs = json.loads(subprocess.check_output(cmd))

		generated_file = """# generated by pkgs/applications/editors/vim/plugins/nvim-treesitter/update.py

		@@ -68,14 +71,27 @@ def update_grammars(lockfile: str):

		{
		"""
		for generated in ThreadPoolExecutor().map(generate_grammar, lockfile.items()):

		lockfile_path = os.path.join(nvim_treesitter_dir, "lockfile.json")
		log.debug("Opening %s", lockfile_path)
		with open(lockfile_path) as lockfile_fd:
		lockfile = json.load(lockfile_fd)

		def _generate_grammar(item):
		lang, lock = item
		cfg = configs.get(lang)
		if not cfg:
		return ""
		return generate_grammar(lang, lock["revision"], cfg)

		for generated in ThreadPoolExecutor(max_workers=5).map(
		_generate_grammar, lockfile.items()
		):
		generated_file += generated
		generated_file += "}\n"
		generated_file += "}\n"

		open(join(dirname(__file__), "generated.nix"), "w").write(generated_file)
		return generated_file


		if __name__ == "__main__":
		# TODO add lockfile
		update_grammars()
		generated = update_grammars(sys.args[1])
		open(join(os.path.dirname(__file__), "generated.nix"), "w").write(generated)

pkgs/applications/editors/vim/plugins/update.py

+20 −13

Original line number	Diff line number	Diff line
		@@ -23,11 +23,12 @@ import os
		import logging
		import textwrap
		import json
		import subprocess
		from typing import List, Tuple
		from pathlib import Path


		log = logging.getLogger()
		log = logging.getLogger("vim-updater")

		sh = logging.StreamHandler()
		formatter = logging.Formatter("%(name)s:%(levelname)s: %(message)s")
		@@ -39,15 +40,14 @@ ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe
		import pluginupdate
		import importlib
		from pluginupdate import run_nix_expr, PluginDesc
		from treesitter import update_grammars
		import treesitter


		HEADER = (
		"# GENERATED by ./pkgs/applications/editors/vim/plugins/update.py. Do not edit!"
		)

		NIXPKGS_NVIMTREESITTER_FOLDER = \
		"pkgs/applications/editors/vim/plugins/nvim-treesitter/generated.nix"
		NIXPKGS_NVIMTREESITTER_FOLDER = "pkgs/applications/editors/vim/plugins/nvim-treesitter"


		class VimEditor(pluginupdate.Editor):
		@@ -58,8 +58,7 @@ class VimEditor(pluginupdate.Editor):
		):
		sorted_plugins = sorted(plugins, key=lambda v: v[0].name.lower())
		nvim_treesitter_rev = pluginupdate.run_nix_expr(
		"(import <localpkgs> { }).vimPlugins.nvim-treesitter.src.rev",
		self.nixpkgs
		"(import <localpkgs> { }).vimPlugins.nvim-treesitter.src.rev", self.nixpkgs
		)

		with open(outfile, "w+") as f:
		@@ -78,7 +77,8 @@ class VimEditor(pluginupdate.Editor):
		content = self.plugin2nix(pdesc, plugin)
		f.write(content)
		if (
		plugin.name == "nvim-treesitter" and plugin.commit != nvim_treesitter_rev
		plugin.name == "nvim-treesitter"
		and plugin.commit != nvim_treesitter_rev
		):
		self.nvim_treesitter_updated = True
		f.write("\n}\n")
		@@ -126,13 +126,19 @@ class VimEditor(pluginupdate.Editor):
		def update(self, args):
		pluginupdate.update_plugins(self, args)

		# TODO this should probably be skipped when running outside a nixpkgs checkout
		if self.nvim_treesitter_updated:
		print("updating nvim-treesitter grammars")
		nvim_treesitter_dir = ROOT.joinpath("nvim-treesitter")
		lockfile = os.path.join(args.nixpkgs.join(NIXPKGS_NVIMTREESITTER_FOLDER, "lockfile.json"))
		lockfile = json.load(open(lockfile))
		cmd = [
		"nix", "build",
		"vimPlugins.nvim-treesitter.src", "-f", self.nixpkgs
		, "--print-out-paths"
		]
		log.debug("Running command: %s", " ".join(cmd))
		nvim_treesitter_dir = subprocess.check_output(cmd, text=True, timeout=90).strip()

		nvim_treesitter.update_grammars(lockfile)
		generated = treesitter.update_grammars(nvim_treesitter_dir)
		open(os.path.join(args.nixpkgs, "generated.nix"), "w").write(generated)

		if self.nixpkgs_repo:
		index = self.nixpkgs_repo.index
		@@ -147,13 +153,14 @@ class VimEditor(pluginupdate.Editor):


		def main():

		global luaPlugins

		log.debug(f"Loading from {ROOT}/../get-plugins.nix")
		with open(f"{ROOT}/../get-plugins.nix") as f:
		GET_PLUGINS = f.read()
		editor = VimEditor("vim", Path("pkgs/applications/editors/vim/plugins"), GET_PLUGINS)
		editor = VimEditor(
		"vim", Path("pkgs/applications/editors/vim/plugins"), GET_PLUGINS
		)
		editor.run()

pkgs/applications/editors/vim/plugins/updater.nix

+3 −7

Original line number	Diff line number	Diff line
		@@ -4,17 +4,12 @@
		, python3Packages
		, lib
		, nix-prefetch-git
		, nurl

		# optional
		, vimPlugins
		, neovim
		}:
		let
		my_neovim = neovim.override {
		configure.packages.all.start = [ vimPlugins.nvim-treesitter ];
		};

		in
		buildPythonApplication {
		format = "other";
		pname = "vim-plugins-updater";
		@@ -39,7 +34,8 @@ buildPythonApplication {
		cp ${../../../../../maintainers/scripts/pluginupdate.py} $out/lib/pluginupdate.py

		# wrap python scripts
		makeWrapperArgs+=( --prefix PATH : "${lib.makeBinPath [ nix nix-prefetch-git my_neovim ]}" --prefix PYTHONPATH : "$out/lib" )
		makeWrapperArgs+=( --prefix PATH : "${lib.makeBinPath [
		nix nix-prefetch-git neovim nurl ]}" --prefix PYTHONPATH : "$out/lib" )
		wrapPythonPrograms
		'';