Unverified Commit fdbb2004 authored by Dmitry Kalinkin's avatar Dmitry Kalinkin Committed by GitHub
Browse files

Merge pull request #247550 from xworld21/build-texlive-package 

texlive: refactor package builder in separate expression
parents 95d630c6 a69d70d6

pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix

0 → 100644
+140 −0
Original line number Diff line number Diff line 
{ lib

, fetchurl

, runCommand



  # script interpreters

, bash

, jdk

, perl

, python3

, ruby

, snobol4

, tk



  # TeX Live prerequisites

, texliveBinaries

}:



{ pname

, revision

, version ? toString revision

, sha512

, mirrors

, extraVersion ? ""

, fixedHashes ? { }

, postUnpack ? ""

, stripPrefix ? 1

, license ? [ ]

, hasHyphens ? false

, hasManpages ? false

, hasRunfiles ? false

, hasTlpkg ? false

, ...

}@args:



let

  meta = { license = map (x: lib.licenses.${x}) license; };



  commonPassthru = {

    inherit pname revision version;

  } // lib.optionalAttrs (args ? extraRevision) {

    inherit (args) extraRevision;

  };



  # build run, doc, source, tlpkg containers

  mkContainer = tlType: passthru: sha512:

    let

      # NOTE: the fixed naming scheme must match generated-fixed-hashes.nix

      # the basename used by upstream (without ".tar.xz" suffix)

      urlName = pname + (lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}");

      # name + version for the derivation

      tlName = urlName + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";

      fixedHash = fixedHashes.${tlType} or null; # be graceful about missing hashes



      urls = args.urls or (if args ? url then [ args.url ] else

      map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") mirrors);

    in

    runCommand "texlive-${tlName}"

      ({

        src = fetchurl { inherit urls sha512; };

        inherit meta passthru stripPrefix tlType;

      } // lib.optionalAttrs (fixedHash != null) {

        outputHash = fixedHash;

        outputHashAlgo = "sha256";

        outputHashMode = "recursive";

      })

      (''

        mkdir "$out"

        if [[ "$tlType"  == "tlpkg" ]]; then

          tar -xf "$src" \

            --strip-components=1 \

            -C "$out" --anchored --exclude=tlpkg/tlpobj --keep-old-files \

            tlpkg

        else

          tar -xf "$src" \

            --strip-components="$stripPrefix" \

            -C "$out" --anchored --exclude=tlpkg --keep-old-files

        fi

      '' + postUnpack);



  tex = [

    (

      let passthru = commonPassthru

        // lib.optionalAttrs (args ? deps) { tlDeps = args.deps; }

        // lib.optionalAttrs (args ? formats) { inherit (args) formats; }

        // lib.optionalAttrs hasHyphens { inherit hasHyphens; }; in

      if hasRunfiles then mkContainer "run" passthru sha512.run

      else (passthru // { tlType = "run"; })

    )

  ];



  doc = let passthru = commonPassthru

    // lib.optionalAttrs hasManpages { inherit hasManpages; }; in

    lib.optional (sha512 ? doc) (mkContainer "doc" passthru sha512.doc);



  source = lib.optional (sha512 ? source) (mkContainer "source" commonPassthru sha512.source);



  tlpkg = let passthru = commonPassthru

    // lib.optionalAttrs (args ? postactionScript) { postactionScript = args.postactionScript; }; in

    lib.optional hasTlpkg (mkContainer "tlpkg" passthru sha512.run);



  bin = lib.optional (args ? binfiles && args.binfiles != [ ]) (

    let

      # find interpreters for the script extensions found in tlpdb

      extToInput = {

        jar = jdk;

        lua = texliveBinaries.luatex;

        py = python3;

        rb = ruby;

        sno = snobol4;

        tcl = tk;

        texlua = texliveBinaries.luatex;

        tlu = texliveBinaries.luatex;

      };

      run = lib.head tex;

    in

    runCommand "texlive-${pname}.bin-${version}"

      {

        passthru = commonPassthru // { tlType = "bin"; };

        inherit meta;

        # shebang interpreters

        buildInputs = (args.extraBuildInputs or [ ]) ++ [ bash perl ] ++ (lib.attrVals (args.scriptExts or [ ]) extToInput);

        # absolute scripts folder

        scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + args.scriptsFolder or pname);

        # binaries info

        inherit (args) binfiles;

        binlinks = builtins.attrNames (args.binlinks or { });

        bintargets = builtins.attrValues (args.binlinks or { });

        binfolders = [ (lib.getBin texliveBinaries.core) ] ++

          lib.optional (texliveBinaries ? ${pname}) (lib.getBin texliveBinaries.${pname});

        # build scripts

        patchScripts = ./patch-scripts.sed;

        makeBinContainers = ./make-bin-containers.sh;

      }

      ''

        . "$makeBinContainers"

        ${args.postFixup or ""}

      ''

  );

in

{ pkgs = tex ++ doc ++ source ++ tlpkg ++ bin; }

pkgs/tools/typesetting/tex/texlive/default.nix

+21 −138
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ let 
  tlpdbVersion = tlpdb."00texlive.config";



  # the set of TeX Live packages, collections, and schemes; using upstream naming

  tl = let

  overriddenTlpdb = let

    # most format -> engine links are generated by texlinks according to fmtutil.cnf at combine time

    # so we remove them from binfiles, and add back the ones texlinks purposefully ignore (e.g. mptopdf)

    removeFormatLinks = lib.mapAttrs (_: attrs:
@@ -43,9 +43,9 @@ let 
           in if binNotFormats != [] then attrs // { binfiles = binNotFormats; } else removeAttrs attrs [ "binfiles" ]

      else attrs);



    orig = removeFormatLinks (removeAttrs tlpdb [ "00texlive.config" ]);

    orig = removeFormatLinks (removeAttrs tlpdb [ "00texlive.config" ]); in



    overridden = lib.recursiveUpdate orig rec {

    lib.recursiveUpdate orig rec {

      #### overrides of texlive.tlpdb



      #### nonstandard script folders
@@ -355,8 +355,8 @@ let 
      # hardcode revision numbers (since texlive.infra, tlshell are not in either system or user texlive.tlpdb)

      tlshell.postFixup = ''

        substituteInPlace "$out"/bin/tlshell \

          --replace '[dict get $::pkgs texlive.infra localrev]' '${toString overridden."texlive.infra".revision}' \

          --replace '[dict get $::pkgs tlshell localrev]' '${toString overridden.tlshell.revision}'

          --replace '[dict get $::pkgs texlive.infra localrev]' '${toString orig."texlive.infra".revision}' \

          --replace '[dict get $::pkgs tlshell localrev]' '${toString orig.tlshell.revision}'

      '';

      #### dependency changes
@@ -380,7 +380,7 @@ let 
      npp-for-context.license = [  "gpl3Only" ];



      texdoc = {

        extraRevision = ".tlpdb${toString tlpdbVersion.revision}";

        extraRevision = "-tlpdb${toString tlpdbVersion.revision}";

        extraVersion = "-tlpdb-${toString tlpdbVersion.revision}";



        # build Data.tlpdb.lua (part of the 'tlType == "run"' package)
@@ -428,39 +428,6 @@ let 
      };

    }; # overrides



    in lib.mapAttrs mkTLPkg overridden;



  # create a TeX package: an attribute set { pkgs = [ ... ]; ... } where pkgs is a list of derivations

  mkTLPkg = pname: attrs:

    let

      version = attrs.version or (toString attrs.revision);

      mkPkgV = tlType: let

        pkg = attrs // {

          sha512 = attrs.sha512.${if tlType == "tlpkg" then "run" else tlType};

          inherit pname tlType version;

        } // lib.optionalAttrs (attrs.hasManpages or false) {

          hasManpages = true;

        };

        in mkPkg pkg;

      run = if (attrs.hasRunfiles or false) then mkPkgV "run"

            # the fake derivations are used for filtering of hyphenation patterns and formats

          else ({

            inherit pname version;

            tlType = "run";

            hasHyphens = attrs.hasHyphens or false;

            tlDeps = map (n: tl.${n}) (attrs.deps or []);

          } // lib.optionalAttrs (attrs ? formats) { inherit (attrs) formats; });

    in {

      # TL pkg contains lists of packages: runtime files, docs, sources, tlpkg, binaries

      pkgs =

        # tarball of a collection/scheme itself only contains a tlobj file

        [ run ]

        ++ lib.optional (attrs.sha512 ? doc) (mkPkgV "doc")

        ++ lib.optional (attrs.sha512 ? source) (mkPkgV "source")

        ++ lib.optional (attrs.hasTlpkg or false) (mkPkgV "tlpkg")

        ++ lib.optional (attrs ? binfiles && attrs.binfiles != []) (mkPkgBin pname version run attrs);

    };



  version = {

    # day of the snapshot being taken

    year = "2023";
@@ -477,7 +444,7 @@ let 
  # need to be used instead. Ideally, for the release branches of NixOS we

  # should be switching to the tlnet-final versions

  # (https://tug.org/historic/).

  urlPrefixes = with version; lib.optionals final  [

  mirrors = with version; lib.optionals final  [

    # tlnet-final snapshot; used when texlive.tlpdb is frozen

    # the TeX Live yearly freeze typically happens in mid-March

    "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${toString texliveYear}/tlnet-final"
@@ -491,7 +458,7 @@ let 
  ];



  tlpdbxz = fetchurl {

    urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") urlPrefixes;

    urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") mirrors;

    hash = "sha256-vm7DmkH/h183pN+qt1p1wZ6peT2TcMk/ae0nCXsCoMw=";

  };
@@ -506,102 +473,17 @@ let 
  # map: name -> fixed-output hash

  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixed-hashes.nix);



  # NOTE: the fixed naming scheme must match generated-fixed-hashes.nix

  # name for the URL

  mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";

  # name + revision for the fixed output hashes

  mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";

  # name + version for the derivation

  mkTLName = { tlType, version, extraVersion ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";



  # build tlType == "bin" containers based on `binfiles` in TLPDB

  # see UPGRADING.md for how to keep the list of shebangs up to date

  mkPkgBin = let extToInput = {

      jar = jdk;

      lua = bin.luatex;

      py = python3;

      rb = ruby;

      sno = snobol4;

      tcl = tk;

      texlua = bin.luatex;

      tlu = bin.luatex;

    }; in pname: version: run:

    { binfiles, scriptsFolder ? pname, postFixup ? "", scriptExts ? [], extraBuildInputs ? [], binlinks ? {}, ... }@args:

    runCommand "texlive-${pname}.bin-${version}"

      {

        # metadata for texlive.combine

        passthru = {

          inherit pname version;

          tlType = "bin";

  buildTeXLivePackage = import ./build-texlive-package.nix {

    inherit lib fetchurl runCommand bash jdk perl python3 ruby snobol4 tk;

    texliveBinaries = bin;

  };

        # shebang interpreters

        buildInputs = extraBuildInputs ++ [ bash perl ] ++ (lib.attrVals scriptExts extToInput);

        # absolute scripts folder

        scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + scriptsFolder);

        # binaries info

        inherit binfiles;

        binlinks = builtins.attrNames binlinks;

        bintargets = builtins.attrValues binlinks;

        binfolders = [ (lib.getBin bin.core) ] ++ lib.optional (bin ? ${pname}) (lib.getBin bin.${pname});

        # build scripts

        patchScripts = ./patch-scripts.sed;

        makeBinContainers = ./make-bin-containers.sh;

      }

      ''

        . "$makeBinContainers"

        ${postFixup}

      '';



  # create a derivation that contains an unpacked upstream TL package

  mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, hasManpages ? false, ... }@args:

    let

      # the basename used by upstream (without ".tar.xz" suffix)

      urlName = mkURLName args;

      tlName = mkTLName args;

      fixedHash = fixedHashes.${mkFixedName args} or null; # be graceful about missing hashes



      urls = args.urls or (if args ? url then [ args.url ] else

        map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));



    in runCommand "texlive-${tlName}"

      ( {

          src = fetchurl { inherit urls sha512; };

          meta = {

            license = map (x: lib.licenses.${x}) (args.license or []);

          };

          inherit stripPrefix tlType;

          # metadata for texlive.combine

          passthru = {

            inherit pname tlType revision version extraRevision;

          } // lib.optionalAttrs (tlType == "run" && args ? deps) {

            tlDeps = map (n: tl.${n}) args.deps;

          } // lib.optionalAttrs (tlType == "run") {

            hasHyphens = args.hasHyphens or false;

          } // lib.optionalAttrs (tlType == "tlpkg" && args ? postactionScript) {

            postactionScript = args.postactionScript;

          }

          // lib.optionalAttrs (tlType == "run" && args ? formats) { inherit (args) formats; }

          // lib.optionalAttrs (tlType == "doc" && hasManpages) { hasManpages = true; };

        } // lib.optionalAttrs (fixedHash != null) {

          outputHash = fixedHash;

          outputHashAlgo = "sha256";

          outputHashMode = "recursive";

        }

      )

      ( ''

          mkdir "$out"

          if [[ "$tlType"  == "tlpkg" ]]; then

            tar -xf "$src" \

              --strip-components=1 \

              -C "$out" --anchored --exclude=tlpkg/tlpobj --keep-old-files \

              tlpkg

          else

            tar -xf "$src" \

              --strip-components="$stripPrefix" \

              -C "$out" --anchored --exclude=tlpkg --keep-old-files

          fi

        '' + postUnpack

      );

  tl = lib.mapAttrs (pname: { revision, extraRevision ? "", ... }@args:

    buildTeXLivePackage (args

      # NOTE: the fixed naming scheme must match generate-fixed-hashes.nix

      // { inherit mirrors pname; fixedHashes = fixedHashes."${pname}-${toString revision}${extraRevision}" or { }; }

      // lib.optionalAttrs (args ? deps) { deps = map (n: tl.${n}) (args.deps or [ ]); })

  ) overriddenTlpdb;



  # combine a set of TL packages into a single TL meta-package

  combinePkgs = pkgList: lib.catAttrs "pkg" (
@@ -640,6 +522,7 @@ in 
      # for backward compatibility

      latexindent = lib.findFirst (p: p.tlType == "bin") tl.latexindent.pkgs;

    };



    combine = assert assertions; combine;



    # Pre-defined combined packages for TeX Live schemes,
@@ -652,7 +535,7 @@ in 
          scheme-basic = [ free gfl gpl1Only gpl2 gpl2Plus knuth lgpl21 lppl1 lppl13c mit ofl publicDomain ];

          scheme-context = [ bsd2 bsd3 cc-by-sa-40 free gfl gfsl gpl1Only gpl2 gpl2Plus gpl3 gpl3Plus knuth lgpl2 lgpl21

            lppl1 lppl13c mit ofl publicDomain x11 ];

          scheme-full = [ artistic1-cl8 asl20 bsd2 bsd3 bsdOriginal cc-by-10 cc-by-40 cc-by-sa-10 cc-by-sa-20

          scheme-full = [ artistic1-cl8 artistic2 asl20 bsd2 bsd3 bsdOriginal cc-by-10 cc-by-40 cc-by-sa-10 cc-by-sa-20

            cc-by-sa-30 cc-by-sa-40 cc0 fdl13Only free gfl gfsl gpl1Only gpl2 gpl2Plus gpl3 gpl3Plus isc knuth

            lgpl2 lgpl21 lgpl3 lppl1 lppl12 lppl13a lppl13c mit ofl publicDomain x11 ];

          scheme-gust = [ artistic1-cl8 asl20 bsd2 bsd3 cc-by-40 cc-by-sa-40 cc0 fdl13Only free gfl gfsl gpl1Only gpl2

pkgs/tools/typesetting/tex/texlive/fixed-hashes.nix

+4243 −9896

File changed.

Preview size limit exceeded, changes collapsed.

pkgs/tools/typesetting/tex/texlive/generate-fixed-hashes.nix

+24 −23
Original line number Diff line number Diff line 
with import ../../../../.. { };



with lib; let

  # NOTE: the fixed naming scheme must match default.nix

  # name for the URL

  mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";

  # name + revision for the fixed output hashes

  mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";



  uniqueByName = fods: catAttrs "fod" (genericClosure {

    startSet = map (fod: { key = fod.name; inherit fod; }) fods;

    operator = _: [ ];

  });

  isFod = p: p.tlType != "bin" && isDerivation p;



  # ugly hack to extract combine from collection-latexextra, since it is masked by texlive.combine

  combine = lib.findFirst (p: (lib.head p.pkgs).pname == "combine") { pkgs = [ ]; } (lib.head texlive.collection-latexextra.pkgs).tlDeps;

  all = concatLists (map (p: p.pkgs or []) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ]))) ++ combine.pkgs;

  all = filter (p: p ? pkgs) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ])) ++ [ combine ];

  sorted = sort (a: b: (head a.pkgs).pname < (head b.pkgs).pname) all;

  fods = filter isFod (concatMap (p: p.pkgs or [ ]) all);



  # fixed hashes only for run, doc, source, tlpkg types

  fods = sort (a: b: a.name < b.name) (uniqueByName (filter (p: isDerivation p && p.tlType != "bin") all));



  computeHash = fod: runCommand "${fod.name}-fixed-hash"

  computeHash = fod: runCommand "${fod.pname}-${fod.tlType}-fixed-hash"

    { buildInputs = [ nix ]; inherit fod; }

    ''echo -n "$(nix-hash --base32 --type sha256 "$fod")" >"$out"'';



  hash = fod: fod.outputHash or (builtins.readFile (computeHash fod));

  hashLine = fod: ''

    "${mkFixedName fod}"="${hash fod}";



  hashes = { pkgs }:

    concatMapStrings ({ tlType, ... }@p: lib.optionalString (isFod p) (''${tlType}="${hash p}";'')) pkgs;



  hashLine = { pkgs }@pkg:

    let

      fods = lib.filter isFod pkgs;

      first = lib.head fods;

      # NOTE: the fixed naming scheme must match default.nix

      fixedName = with first; "${pname}-${toString revision}${first.extraRevision or ""}";

    in

    lib.optionalString (fods != [ ]) ''

      ${strings.escapeNixIdentifier fixedName}={${hashes pkg}};

    '';

in

{
@@ -36,6 +37,6 @@ in 
  fixedHashesNix = writeText "fixed-hashes.nix"

    ''

      {

    ${lib.concatMapStrings hashLine fods}}

      ${lib.concatMapStrings hashLine sorted}}

    '';

}