Commit 7620b617 authored by Vincenzo Mantova's avatar Vincenzo Mantova
Browse files

texlive: implement __overrideTeXConfig and withPackage

Implement new 'buildTeXEnv' to leverage multi-output packages and provide
__overrideTeXConfig/withPackages for modifying the configuration or adding
packages. The override mechanism is prefixed until stabilized.
parent a06e0753
Loading
Loading
Loading
Loading
+1 −1
Original line number Diff line number Diff line
@@ -613,7 +613,7 @@ rec {
        correctLicenses = scheme: builtins.foldl'
                (acc: pkg: concatLicenses acc (lib.toList (pkg.meta.license or [])))
                []
                scheme.passthru.packages;
                scheme.passthru.requiredTeXPackages;
        correctLicensesAttrNames = scheme:
          lib.sort lt
            (map licenseToAttrName (correctLicenses scheme));
+170 −51
Original line number Diff line number Diff line
{ lib, buildEnv, runCommand, writeText, makeWrapper, libfaketime, makeFontsConf
, perl, bash, coreutils, gnused, gnugrep, gawk, ghostscript
, bin, tl, toTLPkgList }:
# combine =
args@{
  pkgFilter ? (pkg: pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "core"
                    || pkg.hasManpages or false)
, extraName ? "combined"
, extraVersion ? ""
, ...
{
  # texlive package set
  tl
, bin

, lib
, buildEnv
, libfaketime
, makeFontsConf
, makeWrapper
, runCommand
, writeShellScript
, writeText
, toTLPkgSets
, bash
, perl

  # common runtime dependencies
, coreutils
, gawk
, gnugrep
, gnused
, ghostscript
}:

lib.fix (self: {
  withDocs ? false
, withSources ? false
, requiredTeXPackages ? ps: [ ps.scheme-infraonly ]

### texlive.combine backward compatibility
, __extraName ? "combined"
, __extraVersion ? ""
# emulate the old texlive.combine (e.g. add man pages to main output)
, __combine ? false
# adjust behavior further if called from the texlive.combine wrapper
, __fromCombineWrapper ? false
}@args:

let
  # combine a set of TL packages into a single TL meta-package
  combinePkgs = pkgList: lib.catAttrs "pkg" (
  ### texlive.combine backward compatibility
  # if necessary, convert old style { pkgs = [ ... ]; } packages to attribute sets
  ensurePkgSets = ps: if ! __fromCombineWrapper && builtins.any (p: p ? pkgs && builtins.all (p: p ? tlType) p.pkgs) ps
    then let oldStyle = builtins.partition (p: p ? pkgs && builtins.all (p: p ? tlType) p.pkgs) ps;
      in oldStyle.wrong ++ lib.concatMap toTLPkgSets oldStyle.right
    else ps;

  pkgList = rec {
    # resolve dependencies of the packages that affect the runtime
    all =
      let
      # a TeX package is an attribute set { pkgs = [ ... ]; ... } where pkgs is a list of derivations
      # the derivations make up the TeX package and optionally (for backward compatibility) its dependencies
      tlPkgToSets = drv: map ({ tlType, version ? "", outputName ? "", ... }@pkg: {
          # outputName required to distinguish among bin.core-big outputs
          key = "${pkg.pname or pkg.name}.${tlType}-${version}-${outputName}";
          inherit pkg;
        }) (drv.pkgs or (toTLPkgList drv));
      pkgListToSets = lib.concatMap tlPkgToSets; in
    builtins.genericClosure {
      startSet = pkgListToSets pkgList;
      operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []);
        # order of packages is irrelevant
        packages = builtins.sort (a: b: a.pname < b.pname) (ensurePkgSets (requiredTeXPackages tl));
        runtime = builtins.partition
          (p: p.outputSpecified or false -> builtins.elem (p.tlOutputName or p.outputName) [ "out" "tex" "tlpkg" ])
          packages;
        keySet = p: {
          key = ((p.name or "${p.pname}-${p.version}") + "-" + p.tlOutputName or p.outputName or "");
          inherit p;
          tlDeps = p.tlDeps or (p.requiredTeXPackages or (_: [ ]) [ ]);
        };
      in
      # texlive.combine: the wrapper already resolves all dependencies
      if __fromCombineWrapper then requiredTeXPackages null else
        builtins.catAttrs "p" (builtins.genericClosure {
          startSet = map keySet runtime.right;
          operator = p: map keySet p.tlDeps;
        }) ++ runtime.wrong;

    # group the specified outputs
    specified = builtins.partition (p: p.outputSpecified or false) all;
    specifiedOutputs = builtins.groupBy (p: p.tlOutputName or p.outputName) specified.right;
    otherOutputNames = builtins.catAttrs "key" (builtins.genericClosure {
      startSet = map (key: { inherit key; }) (lib.concatLists (builtins.catAttrs "outputs" specified.wrong));
      operator = _: [ ];
    });
    otherOutputs = lib.genAttrs otherOutputNames (n: builtins.catAttrs n specified.wrong);
    outputsToInstall = builtins.catAttrs "key" (builtins.genericClosure {
      startSet = map (key: { inherit key; })
        ([ "out" ] ++ lib.optional (splitOutputs ? man) "man"
          ++ lib.concatLists (builtins.catAttrs "outputsToInstall" (builtins.catAttrs "meta" specified.wrong)));
      operator = _: [ ];
    });

  pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ];
  pkgList = rec {
    combined = combinePkgs (lib.attrValues pkgSet);
    all = lib.filter pkgFilter combined;
    splitBin = builtins.partition (p: p.tlType == "bin") all;
    bin = splitBin.right;
    nonbin = splitBin.wrong;
    tlpkg = lib.filter (pkg: pkg.tlType == "tlpkg") combined;
    # split binary and tlpkg from tex, texdoc, texsource
    bin = if __fromCombineWrapper
      then builtins.filter (p: p.tlType == "bin") all # texlive.combine: legacy filter
      else otherOutputs.out or [ ] ++ specifiedOutputs.out or [ ];
    tlpkg = if __fromCombineWrapper
      then builtins.filter (p: p.tlType == "tlpkg") all # texlive.combine: legacy filter
      else otherOutputs.tlpkg or [ ] ++ specifiedOutputs.tlpkg or [ ];

    nonbin = if __fromCombineWrapper then builtins.filter (p: p.tlType != "bin" && p.tlType != "tlpkg") all # texlive.combine: legacy filter
      else (if __combine then # texlive.combine: emulate old input ordering to avoid rebuilds
        lib.concatMap (p: lib.optional (p ? tex) p.tex
          ++ lib.optional ((withDocs || p ? man) && p ? texdoc) p.texdoc
          ++ lib.optional (withSources && p ? texsource) p.texsource) specified.wrong
        else otherOutputs.tex or [ ]
          ++ lib.optionals withDocs (otherOutputs.texdoc or [ ])
          ++ lib.optionals withSources (otherOutputs.texsource or [ ]))
        ++ specifiedOutputs.tex or [ ] ++ specifiedOutputs.texdoc or [ ] ++ specifiedOutputs.texsource or [ ];

    # outputs that do not become part of the environment
    nonEnvOutputs = lib.subtractLists [ "out" "tex" "texdoc" "texsource" "tlpkg" ] otherOutputNames;
  };

  # list generated by inspecting `grep -IR '\([^a-zA-Z]\|^\)gs\( \|$\|"\)' "$TEXMFDIST"/scripts`
  # and `grep -IR rungs "$TEXMFDIST"`
  # and ignoring luatex, perl, and shell scripts (those must be patched using postFixup)
  needsGhostscript = lib.any (p: lib.elem p.pname [ "context" "dvipdfmx" "latex-papersize" "lyluatex" ]) pkgList.bin;

  name = "texlive-${extraName}-${bin.texliveYear}${extraVersion}";
  name = if __combine then "texlive-${__extraName}-${bin.texliveYear}${__extraVersion}" # texlive.combine: old name name
    else "texlive-${bin.texliveYear}-env";

  texmfdist = (buildEnv {
    name = "${name}-texmfdist";

    # remove fake derivations (without 'outPath') to avoid undesired build dependencies
    paths = lib.catAttrs "outPath" pkgList.nonbin;
    paths = builtins.catAttrs "outPath" pkgList.nonbin;

    # mktexlsr
    nativeBuildInputs = [ tl."texlive.infra" ];
@@ -61,7 +131,7 @@ let
    name = "${name}-tlpkg";

    # remove fake derivations (without 'outPath') to avoid undesired build dependencies
    paths = lib.catAttrs "outPath" pkgList.tlpkg;
    paths = builtins.catAttrs "outPath" pkgList.tlpkg;
  }).overrideAttrs (_: { allowSubstitutes = true; });

  # the 'non-relocated' packages must live in $TEXMFROOT/texmf-dist
@@ -74,7 +144,7 @@ let
    ln -s "$tlpkg" "$out"/tlpkg
  '';

  # expose info and man pages in usual /share/{info,man} location
  # texlive.combine: expose info and man pages in usual /share/{info,man} location
  doc = buildEnv {
    name = "${name}-doc";

@@ -87,14 +157,67 @@ let
    ];
  };

in (buildEnv {
  meta = {
    description = "TeX Live environment"
      + lib.optionalString withDocs " with documentation"
      + lib.optionalString (withDocs && withSources) " and"
      + lib.optionalString withSources " with sources";
    platforms = lib.platforms.all;
    longDescription = "Contains the following packages and their transitive dependencies:\n - "
      + lib.concatMapStringsSep "\n - "
          (p: p.pname + (lib.optionalString (p.outputSpecified or false) " (${p.tlOutputName or p.outputName})"))
          (requiredTeXPackages tl);
  };

  # emulate split output derivation
  splitOutputs = {
    out = out // { outputSpecified = true; };
    texmfdist = texmfdist // { outputSpecified = true; };
    texmfroot = texmfroot // { outputSpecified = true; };
  } // (lib.genAttrs pkgList.nonEnvOutputs (outName: (buildEnv {
    inherit name;
    paths = builtins.catAttrs "outPath"
      (pkgList.otherOutputs.${outName} or [ ] ++ pkgList.specifiedOutputs.${outName} or [ ]);
    # force the output to be ${outName} or nix-env will not work
    nativeBuildInputs = [ (writeShellScript "force-output.sh" ''
      export out="''${${outName}-}"
    '') ];
    inherit meta passthru;
  }).overrideAttrs { outputs = [ outName ]; } // { outputSpecified = true; }));

  passthru = lib.optionalAttrs (! __combine) (splitOutputs // {
    all = builtins.attrValues splitOutputs;
    outputs = [ "out" ] ++ pkgList.nonEnvOutputs;
  }) // {
    # This is set primarily to help find-tarballs.nix to do its job
    requiredTeXPackages = builtins.filter lib.isDerivation (pkgList.bin ++ pkgList.nonbin
      ++ lib.optionals (! __fromCombineWrapper)
        (lib.concatMap (n: (pkgList.otherOutputs.${n} or [ ] ++ pkgList.specifiedOutputs.${n} or [ ]))) pkgList.nonEnvOutputs);
    # useful for inclusion in the `fonts.packages` nixos option or for use in devshells
    fonts = "${texmfroot}/texmf-dist/fonts";
    # support variants attrs, (prev: attrs)
    __overrideTeXConfig = newArgs:
      let appliedArgs = if builtins.isFunction newArgs then newArgs args else newArgs; in
        self (args // { __fromCombineWrapper = false; } // appliedArgs);
    withPackages = reqs: self (args // { requiredTeXPackages = ps: requiredTeXPackages ps ++ reqs ps; __fromCombineWrapper = false; });
  };

  out = (if (! __combine)
    # meta.outputsToInstall = [ "out" "man" ] is invalid within buildEnv:
    # checkMeta will notice that there is no actual "man" output, and fail
    # so we set outputsToInstall from the outside, where it is safe
    then lib.addMetaAttrs { inherit (pkgList) outputsToInstall; }
    else x: x) # texlive.combine: man pages used to be part of out
# no indent for git diff purposes
((buildEnv {

  inherit name;

  ignoreCollisions = false;

  # remove fake derivations (without 'outPath') to avoid undesired build dependencies
  paths = lib.catAttrs "outPath" pkgList.bin ++ [ doc ];
  paths = builtins.catAttrs "outPath" pkgList.bin
    ++ lib.optional __combine doc;
  pathsToLink = [
    "/"
    "/share/texmf-var/scripts"
@@ -113,12 +236,7 @@ in (buildEnv {
    perl
  ];

  passthru = {
    # This is set primarily to help find-tarballs.nix to do its job
    packages = lib.filter lib.isDerivation pkgList.all;
    # useful for inclusion in the `fonts.packages` nixos option or for use in devshells
    fonts = "${texmfroot}/texmf-dist/fonts";
  };
  inherit meta passthru;

  postBuild =
    # environment variables (note: only export the ones that are used in the wrappers)
@@ -131,7 +249,7 @@ in (buildEnv {
    export TEXMFCNF="$TEXMFSYSVAR/web2c"
  '' +
    # wrap executables with required env vars as early as possible
    # 1. we want texlive.combine to use the wrapped binaries, to catch bugs
    # 1. we use the wrapped binaries in the scripts below, to catch bugs
    # 2. we do not want to wrap links generated by texlinks
  ''
    enable -f '${bash}/lib/bash/realpath' realpath
@@ -195,16 +313,16 @@ in (buildEnv {
  '' +
    # now filter hyphenation patterns and formats
  (let
    hyphens = lib.filter (p: p.hasHyphens or false && p.tlType == "run") pkgList.splitBin.wrong;
    hyphens = lib.filter (p: p.hasHyphens or false && p.tlOutputName or p.outputName == "tex") pkgList.nonbin;
    hyphenPNames = map (p: p.pname) hyphens;
    formats = lib.filter (p: p ? formats && p.tlType == "run") pkgList.splitBin.wrong;
    formats = lib.filter (p: p ? formats && p.tlOutputName or p.outputName == "tex") pkgList.nonbin;
    formatPNames = map (p: p.pname) formats;
    # sed expression that prints the lines in /start/,/end/ except for /end/
    section = start: end: "/${start}/,/${end}/{ /${start}/p; /${end}/!p; };\n";
    script =
      writeText "hyphens.sed" (
        # document how the file was generated (for language.dat)
        "1{ s/^(% Generated by .*)$/\\1, modified by texlive.combine/; p; }\n"
        "1{ s/^(% Generated by .*)$/\\1, modified by ${if __combine then "texlive.combine" else "Nixpkgs"}/; p; }\n"
        # pick up the header
        + "2,/^% from/{ /^% from/!p; };\n"
        # pick up all sections matching packages that we combine
@@ -214,7 +332,7 @@ in (buildEnv {
      );
    scriptLua =
      writeText "hyphens.lua.sed" (
        "1{ s/^(-- Generated by .*)$/\\1, modified by texlive.combine/; p; }\n"
        "1{ s/^(-- Generated by .*)$/\\1, modified by ${if __combine then "texlive.combine" else "Nixpkgs"}/; p; }\n"
        + "2,/^-- END of language.us.lua/p;\n"
        + lib.concatMapStrings (pname: section "^-- from ${pname}:$" "^}$|^-- from") hyphenPNames
        + "$p;\n"
@@ -225,7 +343,7 @@ in (buildEnv {
    fmtutilSed =
      writeText "fmtutil.sed" (
        # document how file was generated
        "1{ s/^(# Generated by .*)$/\\1, modified by texlive.combine/; }\n"
        "1{ s/^(# Generated by .*)$/\\1, modified by ${if __combine then "texlive.combine" else "Nixpkgs"}/; }\n"
        # disable all formats, even those already disabled
        + "s/^([^#]|#! )/#! \\1/;\n"
        # enable the formats from the packages being installed
@@ -312,4 +430,5 @@ in (buildEnv {
    ln -s "$TEXMFDIST" "$out"/share/texmf
  ''
  ;
}).overrideAttrs (_: { allowSubstitutes = true; })
}).overrideAttrs (_: { allowSubstitutes = true; }));
in out)
+42 −0
Original line number Diff line number Diff line
# legacy texlive.combine wrapper
{ lib, toTLPkgList, toTLPkgSets, buildTeXEnv }:
args@{
  pkgFilter ? (pkg: pkg.tlType == "run" || pkg.tlType == "bin" || pkg.pname == "core"
                    || pkg.hasManpages or false)
, extraName ? "combined"
, extraVersion ? ""
, ...
}:
let
  pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ];

  # combine a set of TL packages into a single TL meta-package
  combinePkgs = pkgList: lib.catAttrs "pkg" (
    let
      # a TeX package used to be an attribute set { pkgs = [ ... ]; ... } where pkgs is a list of derivations
      # the derivations make up the TeX package and optionally (for backward compatibility) its dependencies
      tlPkgToSets = drv: map ({ tlType, version ? "", outputName ? "", ... }@pkg: {
          # outputName required to distinguish among bin.core-big outputs
          key = "${pkg.pname or pkg.name}.${tlType}-${version}-${outputName}";
          inherit pkg;
        }) (drv.pkgs or (toTLPkgList drv));
      pkgListToSets = lib.concatMap tlPkgToSets; in
    builtins.genericClosure {
      startSet = pkgListToSets pkgList;
      operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []);
    });
  combined = combinePkgs (lib.attrValues pkgSet);

  # convert to specified outputs
  tlTypeToOut = { run = "tex"; doc = "texdoc"; source = "texsource"; bin = "out"; tlpkg = "tlpkg"; };
  toSpecified = { tlType, ... }@drv: drv // { outputSpecified = true; tlOutputName = tlTypeToOut.${tlType}; };
  all = lib.filter pkgFilter combined ++ lib.filter (pkg: pkg.tlType == "tlpkg") combined;
  converted = builtins.map toSpecified all;
in
buildTeXEnv {
  __extraName = extraName;
  __extraVersion = extraVersion;
  requiredTeXPackages = _: converted;
  __combine = true;
  __fromCombineWrapper = true;
}
+84 −54
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@
  - source: ../../../../../doc/languages-frameworks/texlive.xml
  - current html: https://nixos.org/nixpkgs/manual/#sec-language-texlive
*/
{ stdenv, lib, fetchurl, runCommand, writeText, buildEnv
{ stdenv, lib, fetchurl, runCommand, writeShellScript, writeText, buildEnv
, callPackage, ghostscript_headless, harfbuzz
, makeWrapper, installShellFiles
, python3, ruby, perl, tk, jdk, bash, snobol4
@@ -22,13 +22,6 @@ let
    tlpdb = overriddenTlpdb;
  };

  # function for creating a working environment from a set of TL packages
  combine = import ./combine.nix {
    inherit bin buildEnv lib makeWrapper writeText runCommand toTLPkgList
      perl libfaketime makeFontsConf bash tl coreutils gawk gnugrep gnused;
    ghostscript = ghostscript_headless;
  };

  tlpdb = import ./tlpdb.nix;

  tlpdbVersion = tlpdb."00texlive.config";
@@ -101,6 +94,14 @@ let
      // lib.optionalAttrs (args ? deps) { deps = map (n: tl.${n}) (args.deps or [ ]); })
  ) overriddenTlpdb;

  # function for creating a working environment
  buildTeXEnv = import ./build-tex-env.nix {
    inherit bin tl;
    ghostscript = ghostscript_headless;
    inherit lib buildEnv libfaketime makeFontsConf makeWrapper runCommand
      writeShellScript writeText toTLPkgSets bash perl coreutils gawk gnugrep gnused;
  };

  ### texlive.combine compatibility layer:
  # convert TeX packages to { pkgs = [ ... ]; } lists
  # respecting specified outputs
@@ -114,34 +115,35 @@ let
      lib.optional (drv ? out) (drv.out // { tlType = "bin"; });
  tlOutToType = { out = "bin"; tex = "run"; texsource = "source"; texdoc = "doc"; tlpkg = "tlpkg"; };

  # convert { pkgs = [ ... ]; } lists to TeX packages
  # possibly more than one, if pkgs is also used to specify dependencies
  tlTypeToOut = { run = "tex"; doc = "texdoc"; source = "texsource"; bin = "out"; tlpkg = "tlpkg"; };
  toSpecifiedNV = p: rec {
    name = value.tlOutputName;
    value = builtins.removeAttrs p [ "pkgs" ]
      // { outputSpecified = true; tlOutputName = tlTypeToOut.${p.tlType}; };
  };
  toTLPkgSet = pname: drvs:
    let set = lib.listToAttrs (builtins.map toSpecifiedNV drvs);
        mainDrv = set.out or set.tex or set.tlpkg or set.texdoc or set.texsource; in
    builtins.removeAttrs mainDrv [ "outputSpecified" ];
  toTLPkgSets = { pkgs, ... }: lib.mapAttrsToList toTLPkgSet
    (builtins.groupBy (p: p.pname) pkgs);

  # export TeX packages as { pkgs = [ ... ]; } in the top attribute set
  allPkgLists = lib.mapAttrs (n: drv: { pkgs = toTLPkgList drv; }) tl;

  # function for creating a working environment from a set of TL packages
  # now a legacy wrapper around buildTeXEnv
  combine = import ./combine-wrapper.nix { inherit buildTeXEnv lib toTLPkgList toTLPkgSets; };

  assertions = with lib;
    assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" &&
    assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate";

in
  allPkgLists // {
    pkgs = tl;

    tlpdb = {
      # nested in an attribute set to prevent them from appearing in search
      nix = tlpdbNix;
      xz = tlpdbxz;
    };

    bin = assert assertions; bin // {
      # for backward compatibility
      latexindent = tl.latexindent;
    };

    combine = assert assertions; combine;

    # Pre-defined combined packages for TeX Live schemes,
  # Pre-defined evironment packages for TeX Live schemes,
  # to make nix-env usage more comfortable and build selected on Hydra.
    combined = with lib;
      let

  # these license lists should be the sorted union of the licenses of the packages the schemes contain.
  # The correctness of this collation is tested by tests.texlive.licenses
  licenses = with lib.licenses; {
@@ -164,24 +166,52 @@ in
      fdl13Only free gfl gpl1Only gpl2 gpl2Plus gpl3 gpl3Plus isc knuth lgpl2 lgpl21 lgpl3 lppl1 lppl12 lppl13a
      lppl13c mit ofl publicDomain x11];
  };
      in recurseIntoAttrs (
      mapAttrs
        (pname: attrs:
          addMetaAttrs rec {
            description = "TeX Live environment for ${pname}";

  meta = {
    description = "TeX Live environment";
    platforms = lib.platforms.all;
    maintainers = with lib.maintainers;  [ veprbl ];
    license = licenses.scheme-infraonly;
  };

  combined = recurseIntoAttrs (
    lib.genAttrs [ "scheme-basic" "scheme-context" "scheme-full" "scheme-gust" "scheme-infraonly"
      "scheme-medium" "scheme-minimal" "scheme-small" "scheme-tetex" ]
      (pname:
        (buildTeXEnv {
          __extraName = "combined" + lib.removePrefix "scheme" pname;
          __extraVersion = with version; if final then "-final" else ".${year}${month}${day}";
          requiredTeXPackages = ps: [ ps.${pname} ];
          # to maintain full backward compatibility, enable texlive.combine behavior
          __combine = true;
        }).overrideAttrs {
          meta = meta // {
            description = "TeX Live environment for ${pname}";
            license = licenses.${pname};
          };
        }
          (combine {
            ${pname} = attrs;
            extraName = "combined" + lib.removePrefix "scheme" pname;
            extraVersion = with version; if final then "-final" else ".${year}${month}${day}";
          })
      )
        { inherit (tl)
            scheme-basic scheme-context scheme-full scheme-gust scheme-infraonly
            scheme-medium scheme-minimal scheme-small scheme-tetex;
        }
  );

in
  allPkgLists // {
    pkgs = tl;

    tlpdb = {
      # nested in an attribute set to prevent them from appearing in search
      nix = tlpdbNix;
      xz = tlpdbxz;
    };

    bin = assert assertions; bin // {
      # for backward compatibility
      latexindent = tl.latexindent;
    };

    combine = assert assertions; combine;

    combined = assert assertions; combined;

    # convenience alias
    withPackages = (buildTeXEnv { }).withPackages;
  }