Merge pull request #206763 from xworld21/texlive-centralize-version

{

  tlpdb-nix = runCommand "texlive-test-tlpdb-nix" {

  tlpdbNix = runCommand "texlive-test-tlpdb-nix" {

    nixpkgsTlpdbNix = ../../tools/typesetting/tex/texlive/tlpdb.nix;

    tlpdbNix = texlive.tlpdb-nix;

    tlpdbNix = texlive.tlpdb.nix;

  }

  ''

    mkdir -p "$out"

# Notes on maintaining/upgrading

## Upgrading texlive.bin

## Upgrading `texlive.bin`

texlive contains a few binaries, defined in bin.nix and released once a year.

`texlive` contains a few binaries, defined in `bin.nix` and released once a year.

In order to reduce closure size for users who just need a few of them, we split it into

packages such as core, core-big, xvdi, etc. This requires making assumptions

packages such as `core`, `core-big`, `xdvi`, etc. This requires making assumptions

about dependencies between the projects that may change between releases; if

you upgrade you may have to do some work here.

## Updating the package set

texlive contains several thousand packages from CTAN, defined in pkgs.nix.

`texlive` contains several thousand packages from CTAN, defined in `tlpdb.nix`.

The CTAN mirrors are not version-controlled and continuously moving,

with more than 100 updates per month.

To create a consistent and reproducible package set in nixpkgs, we snapshot CTAN

and generate nix expressions for all packages in texlive at that point.

We mirror CTAN sources of this snapshot on community-operated servers and on IPFS.

To upgrade the package snapshot, follow this process:

### Snapshot sources and texlive package database

Mirror the current CTAN archive to our mirror(s) and IPFS (URLs in `default.nix`).

See https://tug.org/texlive/acquire-mirror.html for instructions.

To create a consistent and reproducible package set in nixpkgs, we generate nix

expressions for all packages in TeX Live at a certain day.

To upgrade the package snapshot, follow this process.

### Upgrade package information from texlive package database

First, edit `default.nix` as follows.

If upgrading to a daily snapshot:

- change `snapshot.year`, `snapshot.month`, `snapshot.day`;

- ensure `urlPrefixes` uses the https://texlive.info/tlnet-archive mirror;

- ensure `texlive.extraVersion` uses the `snapshot` info.

If upgrading to a final release:

- upgrade `texlive.bin` first;

- ensure `urlPrefixes` uses the historic mirrors;

- ensure `texlive.extraVersion` is `"-final"`.

Then upgrade `tlpdb.hash` to match the new hash of `texlive.tlpdb.xz` and run

Update `version` in `default.nix` with the day of the new snapshot, the new TeX

Live year, and the final status of the snapshot. Then update

`texlive.tlpdbxz.hash` to match the new hash of `texlive.tlpdb.xz` and run

```bash

nix-build ../../../../.. -A texlive.tlpdb-nix --no-out-link

nix-build ../../../../.. -A texlive.tlpdb.nix --no-out-link

```

This will download the daily snapshot of the CTAN package database

`texlive.tlpdb.xz` and extract the relevant package info (including version

numbers and sha512 hashes) for the selected upstream distribution. Then replace

`tlpdb.nix` with the generated file.

This will download either the daily or the final snapshot of the TeX Live

package database `texlive.tlpdb.xz` and extract the relevant package info

(including version numbers and sha512 hashes) for the selected upstream

distribution.

Finally, replace `tlpdb.nix` with the generated file. Note that if the

`version` info does not match the metadata of `tlpdb.nix` (as found in the

`00texlive.config` package), TeX Live packages will not evaluate.

The test `pkgs.tests.texlive.tlpdb-nix` verifies that the file `tlpdb.nix`

The test `pkgs.tests.texlive.tlpdbNix` verifies that the file `tlpdb.nix`

in Nixpkgs matches the one that generated from `texlive.tlpdb.xz`.

### Build packages locally and generate fix hashes

### Commit changes

Commit the updated `pkgs.nix` and `fixedHashes.nix` to the repository.

Commit the updated `tlpdb.nix` and `fixedHashes.nix` to the repository.

let

  withSystemLibs = map (libname: "--with-system-${libname}");

  year = "2022";

  year = toString ((import ./tlpdb.nix)."00texlive.config").year;

  version = year; # keep names simple for now

  common = {

  pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ] // {

    # include a fake "core" package

    core.pkgs = [

      (bin.core.out // { pname = "core"; version = "0"; tlType = "bin"; })

      (bin.core.doc // { pname = "core"; version = "0"; tlType = "doc"; })

      (bin.core.out // { pname = "core"; tlType = "bin"; })

      (bin.core.doc // { pname = "core"; tlType = "doc"; })

    ];

  };

  pkgList = rec {

    ghostscript = ghostscript_headless;

  };

  tlpdb = import ./tlpdb.nix;

  tlpdbVersion = tlpdb."00texlive.config";

  # the set of TeX Live packages, collections, and schemes; using upstream naming

  tl = let

    orig = import ./tlpdb.nix;

    orig = removeAttrs tlpdb [ "00texlive.config" ];

    overridden = orig // {

      # overrides of texlive.tlpdb

      };

      texdoc = orig.texdoc // {

        version = orig.texdoc.version + "-tlpdb-" + (toString tlpdbVersion.revision);

        # build Data.tlpdb.lua (part of the 'tlType == "run"' package)

        postUnpack = ''

          if [[ -f "$out"/scripts/texdoc/texdoc.tlu ]]; then

            unxz --stdout "${tlpdb}" > texlive.tlpdb

            unxz --stdout "${tlpdbxz}" > texlive.tlpdb

            # create dummy doc file to ensure that texdoc does not return an error

            mkdir -p support/texdoc

            ( bin.${pname} // { tlType = "bin"; } );

    };

  # for daily snapshots

  snapshot = {

    year = "2022";

    month = "12";

    day = "27";

  version = {

    # day of the snapshot being taken

    year = "2023";

    month = "03";

    day = "19";

    # TeX Live version

    texliveYear = 2022;

    # final (historic) release or snapshot

    final = true;

  };

  # The tarballs on CTAN mirrors for the current release are constantly

  # need to be used instead. Ideally, for the release branches of NixOS we

  # should be switching to the tlnet-final versions

  # (https://tug.org/historic/).

  urlPrefixes = [

    # tlnet-final snapshot

    "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive"

    "ftp://tug.org/texlive/historic/${bin.texliveYear}/tlnet-final/archive"

    # Daily snapshots hosted by one of the texlive release managers

    #"https://texlive.info/tlnet-archive/${snapshot.year}/${snapshot.month}/${snapshot.day}/tlnet/archive"

  urlPrefixes = with version; lib.optionals final  [

    # tlnet-final snapshot; used when texlive.tlpdb is frozen

    # the TeX Live yearly freeze typically happens in mid-March

    "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${toString texliveYear}/tlnet-final"

    "ftp://tug.org/texlive/historic/${toString texliveYear}/tlnet-final"

  ] ++ [

    # daily snapshots hosted by one of the texlive release managers;

    # used for non-final snapshots and as fallback for final snapshots that have not reached yet the historic mirrors

    # please note that this server is not meant for large scale deployment and should be avoided on release branches

    # https://tug.org/pipermail/tex-live/2019-November/044456.html

    "https://texlive.info/tlnet-archive/${year}/${month}/${day}/tlnet"

  ];

  tlpdb = fetchurl {

    # use the same mirror(s) as urlPrefixes above

    urls = map (up: "${up}/../tlpkg/texlive.tlpdb.xz") urlPrefixes;

  tlpdbxz = fetchurl {

    urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") urlPrefixes;

    hash = "sha256-vm7DmkH/h183pN+qt1p1wZ6peT2TcMk/ae0nCXsCoMw=";

  };

  tlpdb-nix = runCommand "tlpdb.nix" {

    inherit tlpdb;

  tlpdbNix = runCommand "tlpdb.nix" {

    inherit tlpdbxz;

    tl2nix = ./tl2nix.sed;

  }

  ''

    xzcat "$tlpdb" | sed -rn -f "$tl2nix" | uniq > "$out"

    xzcat "$tlpdbxz" | sed -rn -f "$tl2nix" | uniq > "$out"

  '';

  # create a derivation that contains an unpacked upstream TL package

      fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes

      urls = args.urls or (if args ? url then [ args.url ] else

        map (up: "${up}/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));

        map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));

    in runCommand "texlive-${tlName}"

      ( {

      operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []);

    });

  assertions =

    lib.assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" &&

    lib.assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate";

in

  tl // {

    inherit bin combine tlpdb-nix;

    tlpdb = {

      # nested in an attribute set to prevent them from appearing in search

      nix = tlpdbNix;

      xz = tlpdbxz;

    };

    bin = assert assertions; bin;

    combine = assert assertions; combine;

    # Pre-defined combined packages for TeX Live schemes,

    # to make nix-env usage more comfortable and build selected on Hydra.

          (combine {

            ${pname} = attrs;

            extraName = "combined" + lib.removePrefix "scheme" pname;

            #extraVersion = "-final";

            extraVersion = ".${snapshot.year}${snapshot.month}${snapshot.day}";

            extraVersion = with version; if final then "-final" else ".${year}${month}${day}";

          })

        )

        { inherit (tl)