Unverified Commit 700d7ec8 authored by Maximilian Bosch's avatar Maximilian Bosch
Browse files

glibcLocales: re-enable parallel build

This was originally turned off in #245360 because the approach was
non-reproducible, i.e. all N concurrent jobs were racing to update
`locale-archive`.

glibc's localedef allows to separate the build from the install, i.e.
`localedef --no-archive` allows to only perform the build of the locale
which can be parallelized again. After that, a single process adds all
newly built locales into the archive.

This isn't supported by the Makefiles in upstream glibc yet[1], hence
the additional patches.

For me, this reduces the build-time of `pkgs.glibcLocales` from ~7min to
slightly under 1min.

[1] https://sourceware.org/pipermail/libc-alpha/2025-October/170803.html
parent cfb771c8
Loading
Loading
Loading
Loading
+50 −0
Original line number Diff line number Diff line
From 38e9742ed794febbc8caa4177df14d097607a2d9 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 5 Oct 2025 12:45:25 +0200
Subject: [PATCH 1/2] localedata: allow reproducible & parallel install of
 locale-archive

The problem with the current approach is that N processes are compiling
a locale and updating the locale-archive at the same time. Hence,
there's a race which process gets to update the archive first and thus
the build is not reproducible[1].

This patch changes that by adding the `--no-archive` flag to the
`$(LOCALEDEF)` invocation in `build-one-locale` to parallelize the build
process in a race-free manner. A single `localedef --add-to-archive`
then adds all previously built locales to the archive.

[1] https://github.com/NixOS/nixpkgs/issues/245360
---
 localedata/Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/localedata/Makefile b/localedata/Makefile
index 01b32348d4..13f7d3a2c4 100644
--- a/localedata/Makefile
+++ b/localedata/Makefile
@@ -538,8 +538,13 @@ LOCALEDEF=I18NPATH=. GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
 $(rtld-prefix) $(common-objpfx)locale/localedef
 install-locales: install-locale-archive
 
+define install-all-locales
+  $(LOCALEDEF) $$flags --add-to-archive `find $(inst_complocaledir) -maxdepth 1 -mindepth 1 -type d | sort`
+endef
+
 # Create and install the locale-archive file.
 install-locale-archive: $(INSTALL-SUPPORTED-LOCALE-ARCHIVE)
+	$(install-all-locales)
 # Create and install the locales individually (no archive).
 install-locale-files: $(INSTALL-SUPPORTED-LOCALE-FILES)
 
@@ -565,6 +570,7 @@ define build-one-locale
 	$(LOCALEDEF) $$flags --alias-file=../intl/locale.alias \
 		     -i locales/$$input -f charmaps/$$charset \
 		     $(addprefix --prefix=,$(install_root)) $$locale \
+		     --no-archive \
 	&& echo ' done';
 endef
 
-- 
2.50.1
+42 −0
Original line number Diff line number Diff line
From 887cb6906afa2b2d8abf35ccfc33ce7b8dc902f8 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 5 Oct 2025 12:48:43 +0200
Subject: [PATCH 2/2] Makeconfig: make inst_complocaledir overridable

On NixOS, we build the locales in a separate package from glibc itself.
Also, packages aren't installed into a common location such as `/usr`,
but each package has its own unique prefix below `/nix/store`.

For us, the current state is a problem because `localedef` is built
in a different package and expects DESTDIR to be the unique
package-prefix of `glibc` itself. Our current workaround is to
invoke `localedef` with `--prefix=$TMPDIR` and copy the contents
to the correct prefix. This was doable without patching when
`localedef` was doing build & locale-archive creation at the
same time.

With the previous patch, the Makefile uses `$(inst_complocaledir)` to
tell localedef where the previously built locales are. In NixOS, we now
need to override `$(inst_complocaledir)` to point to
`$TMPDIR/$(glibc.out)`. Hence, this patch makes the variable overridable
from env.
---
 Makeconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makeconfig b/Makeconfig
index 9eda4fa528..43d88325bc 100644
--- a/Makeconfig
+++ b/Makeconfig
@@ -197,7 +197,7 @@ inst_zonedir = $(install_root)$(zonedir)
 ifndef complocaledir
 complocaledir = $(libdir)/locale
 endif
-inst_complocaledir = $(install_root)$(complocaledir)
+inst_complocaledir ?= $(install_root)$(complocaledir)
 
 # Where to install the message catalog data files (which are
 # machine-independent).
-- 
2.50.1
+4 −0
Original line number Diff line number Diff line
@@ -111,6 +111,10 @@ stdenv.mkDerivation (
        & https://github.com/NixOS/nixpkgs/pull/188492#issuecomment-1233802991
      */
      ./reenable_DT_HASH.patch

      # enable parallel & reproducible build of glibcLocales
      ./0001-localedata-allow-reproducible-parallel-install-of-lo.patch
      ./0002-Makeconfig-make-inst_complocaledir-overridable.patch
    ]
    /*
      NVCC does not support ARM intrinsics. Since <math.h> is pulled in by almost
+2 −7
Original line number Diff line number Diff line
@@ -61,7 +61,8 @@
          # $TMPDIR/nix/store/...-glibc-.../lib/locale/locale-archive.
          LOCALEDEF_FLAGS+=" --prefix=$TMPDIR"

          mkdir -p $TMPDIR/"${buildPackages.glibc.out}/lib/locale"
          export inst_complocaledir="$TMPDIR/"${buildPackages.glibc.out}/lib/locale
          mkdir -p "$inst_complocaledir"

          echo 'C.UTF-8/UTF-8 \' >> ../glibc-2*/localedata/SUPPORTED

@@ -88,12 +89,6 @@
          echo SUPPORTED-LOCALES='${toString locales}' > ../glibc-2*/localedata/SUPPORTED
        '';

      # Current `nixpkgs` way of building locales is not compatible with
      # parallel install. `locale-archive` is updated in parallel with
      # multiple `localedef` processes and causes non-deterministic result:
      #   https://github.com/NixOS/nixpkgs/issues/245360
      enableParallelBuilding = false;

      makeFlags = (previousAttrs.makeFlags or [ ]) ++ [
        "localedata/install-locales"
        "localedir=${placeholder "out"}/lib/locale"