Commit 9bfc2e20 authored by kb's avatar kb
Browse files

tree-sitter-grammars: refactor source definitions 

Major refactor of the approach to defining tree-sitter grammars. This
provides a clearer abstraction to declare grammar sources and removes
custom update logic in favour of nix-update.
parent 62b21fb4

pkgs/development/tools/parsing/tree-sitter/grammar.nix→pkgs/development/tools/parsing/tree-sitter/build-grammar.nix

+118 −0
Original line number Diff line number Diff line
@@ -2,28 +2,28 @@ 
  stdenv,

  nodejs,

  tree-sitter,

  jq,

  lib,

}:



# Build a parser grammar and put the resulting shared object in `$out/parser`



{

  # language name

  language,

  version,

  src,

  location ? null,

  generate ? false,

  ...

}@args:



stdenv.mkDerivation (

  {

    pname = "${language}-grammar";

    pname = "tree-sitter-${language}";



    inherit src version;

    inherit version src;



    nativeBuildInputs = lib.optionals generate [

    nativeBuildInputs = [

      jq

    ]

    ++ lib.optionals generate [

      nodejs

      tree-sitter

    ];
@@ -39,11 +39,45 @@ stdenv.mkDerivation ( 


    stripDebugList = [ "parser" ];



    configurePhase =

      lib.optionalString (location != null) ''

        cd ${location}

      ''

      + lib.optionalString generate ''

    # Tree-sitter grammar packages contain a `tree-sitter.json` file at their

    # root. This provides package metadata that can be used to infer build

    # details.

    #

    # See https://tree-sitter.github.io/tree-sitter/cli/init.html for spec.

    configurePhase = ''

      runHook preConfigure

      if [[ -e tree-sitter.json ]]; then

        # Check nix package version matches grammar source

        NIX_VERSION=${lib.head (lib.splitString "+" version)}

        SRC_VERSION=$(jq -r '.metadata.version' tree-sitter.json)

        if [[ "$NIX_VERSION" != "$SRC_VERSION" ]]; then

          nixErrorLog "grammar version ($NIX_VERSION) differs from source ($SRC_VERSION)"

        fi



        # Check language name matches source

        GRAMMAR=$(jq -c 'first(.grammars[] | select(.name == env.language))' tree-sitter.json)

        if [[ -z "$GRAMMAR" ]]; then

          GRAMMAR=$(jq -c 'first(.grammars[]) // {}' tree-sitter.json)

          NAME=$(jq -r '.name' <<< "$GRAMMAR")

          SRC_LANGS=$(jq -r '[.grammars[].name] | join(", ")' tree-sitter.json)

          nixErrorLog "grammar name ($language) not found in source grammars ($SRC_LANGS), continuing with $NAME"

        fi



        # Move to the appropriate working directory for build

        cd -- $(jq -r '.path // "."' <<< $GRAMMAR)

      else

        # Older grammars may not contain this file. The tree-sitter CLI provides

        # a warning rather than fail unless ABI > 14. Mirror that behaviour

        # while older grammars age out.

        nixWarnLog "grammar source is missing tree-sitter.json"

      fi

      runHook postConfigure

    '';



    # Optionally regenerate the parser source from the defined grammar. In most

    # cases this should not be required as convention is to have this checked

    # in to the source repo.

    preBuild = lib.optionalString generate ''

      tree-sitter generate

    '';
@@ -72,9 +106,13 @@ stdenv.mkDerivation ( 
      runHook postInstall

    '';

  }

  # FIXME: neovim and nvim-treesitter currently rely on passing location rather

  # than a src attribute with a correctly positioned root (e.g. for grammars in

  # monorepos). Use this if present for now.

  // (lib.optionalAttrs (args ? location && args.location != null) {

    setSourceRoot = "sourceRoot=$(echo */${args.location})";

  })

  // removeAttrs args [

    "language"

    "location"

    "generate"

  ]

)

pkgs/development/tools/parsing/tree-sitter/default.nix

+51 −112
Original line number Diff line number Diff line 
{

  lib,

  stdenv,

  fetchgit,

  fetchFromGitHub,

  fetchFromGitLab,

  fetchFromSourcehut,

  nix-update-script,

  runCommand,

  which,
@@ -18,122 +19,55 @@ 
  enableShared ? !stdenv.hostPlatform.isStatic,

  enableStatic ? stdenv.hostPlatform.isStatic,

  webUISupport ? false,

  extraGrammars ? { },



  # tests

  lunarvim,

}:



let

  # to update:

  # 1) change all these hashes

  # 2) nix-build -A tree-sitter.updater.update-all-grammars

  # 3) Set GITHUB_TOKEN env variable to avoid api rate limit (Use a Personal Access Token from https://github.com/settings/tokens It does not need any permissions)

  # 4) run the ./result script that is output by that (it updates ./grammars)

  version = "0.25.10";

  hash = "sha256-aHszbvLCLqCwAS4F4UmM3wbSb81QuG9FM7BDHTu1ZvM=";

  /**

    Build a parser grammar and put the resulting shared object in `$out/parser`.



  src = fetchFromGitHub {

    owner = "tree-sitter";

    repo = "tree-sitter";

    tag = "v${version}";

    inherit hash;

    fetchSubmodules = true;

    # Example



    ```nix

    tree-sitter-foo = pkgs.tree-sitter.buildGrammar {

      language = "foo";

      version = "0.42.0";

      src = fetchFromGitHub { ... };

    };

    ```

  */

  buildGrammar = callPackage ./build-grammar.nix { };



  /**

    Attrset of grammar sources.



  update-all-grammars = callPackage ./update.nix { };

    Values are of the form { language, version, src }. These may be referenced

    directly by other areas of the tree-sitter ecosystem in nixpkgs. Src will

    contain all language bindings provided by the upstream grammar.



  fetchGrammar =

    v:

    fetchgit {

      inherit (v)

        url

        rev

        sha256

        fetchSubmodules

    Use pkgs.tree-sitter.grammars.<name> to access.

  */

  grammars = import ./grammars {

    inherit

      lib

      nix-update-script

      fetchFromGitHub

      fetchFromGitLab

      fetchFromSourcehut

      ;

  };



  grammars = runCommand "grammars" { } (

    ''

      mkdir $out

    ''

    + (lib.concatStrings (

      lib.mapAttrsToList (

        name: grammar: "ln -s ${grammar.src or (fetchGrammar grammar)} $out/${name}\n"

      ) (import ./grammars { inherit lib; })

    ))

  );

  /**

    Attrset of compiled grammars.



  buildGrammar = callPackage ./grammar.nix { };

    Compiled grammars contain the pre-built shared library and any queries from

    the grammar source.



  builtGrammars =

    let

      build =

        name: grammar:

        buildGrammar {

          language = grammar.language or name;

          inherit version;

          src = grammar.src or (fetchGrammar grammar);

          location = grammar.location or null;

          generate = grammar.generate or false;

        };

      grammars' = import ./grammars { inherit lib; } // extraGrammars;

      grammars =

        grammars'

        // {

          tree-sitter-latex = grammars'.tree-sitter-latex // {

            generate = true;

          };

        }

        // {

          tree-sitter-ocaml = grammars'.tree-sitter-ocaml // {

            location = "grammars/ocaml";

          };

        }

        // {

          tree-sitter-ocaml-interface = grammars'.tree-sitter-ocaml // {

            location = "grammars/interface";

          };

        }

        // {

          tree-sitter-org-nvim = grammars'.tree-sitter-org-nvim // {

            language = "tree-sitter-org";

          };

        }

        // {

          tree-sitter-typescript = grammars'.tree-sitter-typescript // {

            location = "typescript";

          };

        }

        // {

          tree-sitter-tsx = grammars'.tree-sitter-typescript // {

            location = "tsx";

          };

        }

        // {

          tree-sitter-markdown = grammars'.tree-sitter-markdown // {

            location = "tree-sitter-markdown";

          };

        }

        // {

          tree-sitter-markdown-inline = grammars'.tree-sitter-markdown // {

            language = "tree-sitter-markdown_inline";

            location = "tree-sitter-markdown-inline";

          };

        }

        // {

          tree-sitter-php = grammars'.tree-sitter-php // {

            location = "php";

          };

        }

        // {

          tree-sitter-sql = grammars'.tree-sitter-sql // {

            generate = true;

          };

        };

    in

    lib.mapAttrs build grammars;

    Use pkgs.tree-sitter-grammars.<name> to access.

  */

  builtGrammars = lib.mapAttrs (_: lib.makeOverridable buildGrammar) grammars;



  # Usage:

  # pkgs.tree-sitter.withPlugins (p: [ p.tree-sitter-c p.tree-sitter-java ... ])
@@ -167,9 +101,17 @@ let 
  allGrammars = builtins.attrValues builtGrammars;



in

rustPlatform.buildRustPackage {

rustPlatform.buildRustPackage (final: {

  pname = "tree-sitter";

  inherit src version;

  version = "0.25.10";



  src = fetchFromGitHub {

    owner = "tree-sitter";

    repo = "tree-sitter";

    tag = "v${final.version}";

    hash = "sha256-aHszbvLCLqCwAS4F4UmM3wbSb81QuG9FM7BDHTu1ZvM=";

    fetchSubmodules = true;

  };



  cargoHash = "sha256-4R5Y9yancbg/w3PhACtsWq0+gieUd2j8YnmEj/5eqkg=";
@@ -229,9 +171,6 @@ rustPlatform.buildRustPackage { 
  doCheck = false;



  passthru = {

    updater = {

      inherit update-all-grammars;

    };

    inherit

      grammars

      buildGrammar
@@ -254,7 +193,7 @@ rustPlatform.buildRustPackage { 
    homepage = "https://github.com/tree-sitter/tree-sitter";

    description = "Parser generator tool and an incremental parsing library";

    mainProgram = "tree-sitter";

    changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}";

    changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${final.version}";

    longDescription = ''

      Tree-sitter is a parser generator tool and an incremental parsing library.

      It can build a concrete syntax tree for a source file and efficiently update the syntax tree as the source file is edited.
@@ -273,4 +212,4 @@ rustPlatform.buildRustPackage { 
      amaanq

    ];

  };

}
 
})

pkgs/development/tools/parsing/tree-sitter/grammars/default.nix

+93 −128
Original line number Diff line number Diff line 
{ lib }:

{

  tree-sitter-bash = lib.importJSON ./tree-sitter-bash.json;

  tree-sitter-beancount = lib.importJSON ./tree-sitter-beancount.json;

  tree-sitter-bibtex = lib.importJSON ./tree-sitter-bibtex.json;

  tree-sitter-bitbake = lib.importJSON ./tree-sitter-bitbake.json;

  tree-sitter-bqn = lib.importJSON ./tree-sitter-bqn.json;

  tree-sitter-c = lib.importJSON ./tree-sitter-c.json;

  tree-sitter-c-sharp = lib.importJSON ./tree-sitter-c-sharp.json;

  tree-sitter-clojure = lib.importJSON ./tree-sitter-clojure.json;

  tree-sitter-cmake = lib.importJSON ./tree-sitter-cmake.json;

  tree-sitter-comment = lib.importJSON ./tree-sitter-comment.json;

  tree-sitter-commonlisp = lib.importJSON ./tree-sitter-commonlisp.json;

  tree-sitter-cpp = lib.importJSON ./tree-sitter-cpp.json;

  tree-sitter-crystal = lib.importJSON ./tree-sitter-crystal.json;

  tree-sitter-css = lib.importJSON ./tree-sitter-css.json;

  tree-sitter-cuda = lib.importJSON ./tree-sitter-cuda.json;

  tree-sitter-cue = lib.importJSON ./tree-sitter-cue.json;

  tree-sitter-dart = lib.importJSON ./tree-sitter-dart.json;

  tree-sitter-devicetree = lib.importJSON ./tree-sitter-devicetree.json;

  tree-sitter-dockerfile = lib.importJSON ./tree-sitter-dockerfile.json;

  tree-sitter-dot = lib.importJSON ./tree-sitter-dot.json;

  tree-sitter-earthfile = lib.importJSON ./tree-sitter-earthfile.json;

  tree-sitter-eex = lib.importJSON ./tree-sitter-eex.json;

  tree-sitter-elisp = lib.importJSON ./tree-sitter-elisp.json;

  tree-sitter-elixir = lib.importJSON ./tree-sitter-elixir.json;

  tree-sitter-elm = lib.importJSON ./tree-sitter-elm.json;

  tree-sitter-embedded-template = lib.importJSON ./tree-sitter-embedded-template.json;

  tree-sitter-erlang = lib.importJSON ./tree-sitter-erlang.json;

  tree-sitter-factor = lib.importJSON ./tree-sitter-factor.json;

  tree-sitter-fennel = lib.importJSON ./tree-sitter-fennel.json;

  tree-sitter-fish = lib.importJSON ./tree-sitter-fish.json;

  tree-sitter-fortran = lib.importJSON ./tree-sitter-fortran.json;

  tree-sitter-gdscript = lib.importJSON ./tree-sitter-gdscript.json;

  tree-sitter-gemini = lib.importJSON ./tree-sitter-gemini.json;

  tree-sitter-gleam = lib.importJSON ./tree-sitter-gleam.json;

  tree-sitter-glimmer = lib.importJSON ./tree-sitter-glimmer.json;

  tree-sitter-glsl = lib.importJSON ./tree-sitter-glsl.json;

  tree-sitter-go = lib.importJSON ./tree-sitter-go.json;

  tree-sitter-go-template = lib.importJSON ./tree-sitter-go-template.json;

  tree-sitter-godot-resource = lib.importJSON ./tree-sitter-godot-resource.json;

  tree-sitter-gomod = lib.importJSON ./tree-sitter-gomod.json;

  tree-sitter-gowork = lib.importJSON ./tree-sitter-gowork.json;

  tree-sitter-graphql = lib.importJSON ./tree-sitter-graphql.json;

  tree-sitter-haskell = lib.importJSON ./tree-sitter-haskell.json;

  tree-sitter-hcl = lib.importJSON ./tree-sitter-hcl.json;

  tree-sitter-heex = lib.importJSON ./tree-sitter-heex.json;

  tree-sitter-hjson = lib.importJSON ./tree-sitter-hjson.json;

  tree-sitter-html = lib.importJSON ./tree-sitter-html.json;

  tree-sitter-http = lib.importJSON ./tree-sitter-http.json;

  tree-sitter-hyprlang = lib.importJSON ./tree-sitter-hyprlang.json;

  tree-sitter-janet-simple = lib.importJSON ./tree-sitter-janet-simple.json;

  tree-sitter-java = lib.importJSON ./tree-sitter-java.json;

  tree-sitter-javascript = lib.importJSON ./tree-sitter-javascript.json;

  tree-sitter-jsdoc = lib.importJSON ./tree-sitter-jsdoc.json;

  tree-sitter-json = lib.importJSON ./tree-sitter-json.json;

  tree-sitter-json5 = lib.importJSON ./tree-sitter-json5.json;

  tree-sitter-jsonnet = lib.importJSON ./tree-sitter-jsonnet.json;

  tree-sitter-julia = lib.importJSON ./tree-sitter-julia.json;

  tree-sitter-just = lib.importJSON ./tree-sitter-just.json;

  tree-sitter-kdl = lib.importJSON ./tree-sitter-kdl.json;

  tree-sitter-koka = lib.importJSON ./tree-sitter-koka.json;

  tree-sitter-kotlin = lib.importJSON ./tree-sitter-kotlin.json;

  tree-sitter-latex = lib.importJSON ./tree-sitter-latex.json;

  tree-sitter-ledger = lib.importJSON ./tree-sitter-ledger.json;

  tree-sitter-llvm = lib.importJSON ./tree-sitter-llvm.json;

  tree-sitter-lua = lib.importJSON ./tree-sitter-lua.json;

  tree-sitter-make = lib.importJSON ./tree-sitter-make.json;

  tree-sitter-markdown = lib.importJSON ./tree-sitter-markdown.json;

  tree-sitter-netlinx = lib.importJSON ./tree-sitter-netlinx.json;

  tree-sitter-nickel = lib.importJSON ./tree-sitter-nickel.json;

  tree-sitter-nix = lib.importJSON ./tree-sitter-nix.json;

  tree-sitter-norg = lib.importJSON ./tree-sitter-norg.json;

  tree-sitter-norg-meta = lib.importJSON ./tree-sitter-norg-meta.json;

  tree-sitter-nu = lib.importJSON ./tree-sitter-nu.json;

  tree-sitter-ocaml = lib.importJSON ./tree-sitter-ocaml.json;

  tree-sitter-org-nvim = lib.importJSON ./tree-sitter-org-nvim.json;

  tree-sitter-perl = lib.importJSON ./tree-sitter-perl.json;

  tree-sitter-pgn = lib.importJSON ./tree-sitter-pgn.json;

  tree-sitter-php = lib.importJSON ./tree-sitter-php.json;

  tree-sitter-pioasm = lib.importJSON ./tree-sitter-pioasm.json;

  tree-sitter-prisma = lib.importJSON ./tree-sitter-prisma.json;

  tree-sitter-proto = lib.importJSON ./tree-sitter-proto.json;

  tree-sitter-pug = lib.importJSON ./tree-sitter-pug.json;

  tree-sitter-python = lib.importJSON ./tree-sitter-python.json;

  tree-sitter-ql = lib.importJSON ./tree-sitter-ql.json;

  tree-sitter-ql-dbscheme = lib.importJSON ./tree-sitter-ql-dbscheme.json;

  tree-sitter-query = lib.importJSON ./tree-sitter-query.json;

  tree-sitter-r = lib.importJSON ./tree-sitter-r.json;

  tree-sitter-regex = lib.importJSON ./tree-sitter-regex.json;

  tree-sitter-rego = lib.importJSON ./tree-sitter-rego.json;

  tree-sitter-river = lib.importJSON ./tree-sitter-river.json;

  tree-sitter-rst = lib.importJSON ./tree-sitter-rst.json;

  tree-sitter-ruby = lib.importJSON ./tree-sitter-ruby.json;

  tree-sitter-rust = lib.importJSON ./tree-sitter-rust.json;

  tree-sitter-scala = lib.importJSON ./tree-sitter-scala.json;

  tree-sitter-scheme = lib.importJSON ./tree-sitter-scheme.json;

  tree-sitter-scss = lib.importJSON ./tree-sitter-scss.json;

  tree-sitter-slint = lib.importJSON ./tree-sitter-slint.json;

  tree-sitter-smithy = lib.importJSON ./tree-sitter-smithy.json;

  tree-sitter-sml = lib.importJSON ./tree-sitter-sml.json;

  tree-sitter-solidity = lib.importJSON ./tree-sitter-solidity.json;

  tree-sitter-sparql = lib.importJSON ./tree-sitter-sparql.json;

  tree-sitter-sql = lib.importJSON ./tree-sitter-sql.json;

  tree-sitter-supercollider = lib.importJSON ./tree-sitter-supercollider.json;

  tree-sitter-surface = lib.importJSON ./tree-sitter-surface.json;

  tree-sitter-svelte = lib.importJSON ./tree-sitter-svelte.json;

  tree-sitter-talon = lib.importJSON ./tree-sitter-talon.json;

  tree-sitter-templ = lib.importJSON ./tree-sitter-templ.json;

  tree-sitter-tera = lib.importJSON ./tree-sitter-tera.json;

  tree-sitter-tiger = lib.importJSON ./tree-sitter-tiger.json;

  tree-sitter-tlaplus = lib.importJSON ./tree-sitter-tlaplus.json;

  tree-sitter-todotxt = lib.importJSON ./tree-sitter-todotxt.json;

  tree-sitter-toml = lib.importJSON ./tree-sitter-toml.json;

  tree-sitter-tsq = lib.importJSON ./tree-sitter-tsq.json;

  tree-sitter-turtle = lib.importJSON ./tree-sitter-turtle.json;

  tree-sitter-twig = lib.importJSON ./tree-sitter-twig.json;

  tree-sitter-typescript = lib.importJSON ./tree-sitter-typescript.json;

  tree-sitter-typst = lib.importJSON ./tree-sitter-typst.json;

  tree-sitter-uiua = lib.importJSON ./tree-sitter-uiua.json;

  tree-sitter-verilog = lib.importJSON ./tree-sitter-verilog.json;

  tree-sitter-vim = lib.importJSON ./tree-sitter-vim.json;

  tree-sitter-vue = lib.importJSON ./tree-sitter-vue.json;

  tree-sitter-wgsl = lib.importJSON ./tree-sitter-wgsl.json;

  tree-sitter-wing = lib.importJSON ./tree-sitter-wing.json;

  tree-sitter-yaml = lib.importJSON ./tree-sitter-yaml.json;

  tree-sitter-yang = lib.importJSON ./tree-sitter-yang.json;

  tree-sitter-zig = lib.importJSON ./tree-sitter-zig.json;

  lib,

  fetchFromGitHub,

  fetchFromGitLab,

  fetchFromSourcehut,

  nix-update-script,

}:



let

  /**

    Set of grammar sources. See ./grammar-sources.nix to define a new grammar.

  */

  grammar-sources = import ./grammar-sources.nix;



  /**

    Parse a flakeref style string to { type, owner, repo, ref }



    FIXME: switch to builtins.parseFlakeRef when stable

  */

  parseUrl =

    url:

    let

      parts = lib.match "(.+):([^/]+)\/([^/?]+)((\/|.+ref=)([^&]+))?" url;

      ref = lib.elemAt parts 5;

    in

    {

      type = lib.elemAt parts 0;

      owner = lib.elemAt parts 1;

      repo = lib.elemAt parts 2;

    }

    // lib.optionalAttrs (ref != null) { inherit ref; };



  /**

    Check if a version attr represents an unstable release.

  */

  isUnstable = version: lib.isList (lib.match ".+-unstable-.+" version);



in



lib.mapAttrs' (

  language: attrs:

  lib.nameValuePair "tree-sitter-${language}" (

    {

      # Default to the source attr name as the language

      inherit language;



      # Insert auto-update support

      passthru.updateScript = nix-update-script {

        extraArgs = [

          "--override-filename"

          "pkgs/development/tools/parsing/tree-sitter/grammars/grammar-sources.nix"

        ]

        ++ (

          if (isUnstable attrs.version) then

            [

              "--version"

              "branch"

            ]

          else

            [

              "--version-regex"

              "^v?(\\d+\\.\\d+\\.\\d+)"

            ]

        );

      };

    }



    # Expand flakeref style shorthand into a source expression

    // lib.optionalAttrs (attrs ? url && attrs ? hash) {

      src =

        let

          source = parseUrl attrs.url;

          fetch = lib.getAttr source.type {

            github = fetchFromGitHub;

            gitlab = fetchFromGitLab;

            sourcehut = fetchFromSourcehut;

            # NOTE: include other types here as required

          };

        in

        fetch {

          inherit (source)

            owner

            repo

            ;

          rev = source.ref or "v${attrs.version}";

          inherit (attrs) hash;

        };

    }

    // removeAttrs attrs [

      "url"

      "hash"

    ]

  )

) grammar-sources

pkgs/development/tools/parsing/tree-sitter/grammars/grammar-sources.nix

0 → 100644
+795 −0

File added.

Preview size limit exceeded, changes collapsed.

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-bash.json

deleted100644 → 0
+0 −12
Original line number Diff line number Diff line 
{

  "url": "https://github.com/tree-sitter/tree-sitter-bash",

  "rev": "487734f87fd87118028a65a4599352fa99c9cde8",

  "date": "2024-11-11T01:52:16-05:00",

  "path": "/nix/store/llqfabr73wh33skh2qzhwjh93zc5cy09-tree-sitter-bash",

  "sha256": "1smlcfkxxknhya1b1h72zj3ccg35szbg9mii2xwh7iq9acnlzpgc",

  "hash": "sha256-7N1PLVMJxwN5FzHW9NbXZTzGhvziwLCC8tDO3qdjtOo=",

  "fetchLFS": false,

  "fetchSubmodules": false,

  "deepClone": false,

  "leaveDotGit": false

}