glibcLocales: re-enable parallel build

This was originally turned off in #245360 because the approach was
non-reproducible, i.e. all N concurrent jobs were racing to update
`locale-archive`.

glibc's localedef allows to separate the build from the install, i.e.
`localedef --no-archive` allows to only perform the build of the locale
which can be parallelized again. After that, a single process adds all
newly built locales into the archive.

This isn't supported by the Makefiles in upstream glibc yet[1], hence
the additional patches.

For me, this reduces the build-time of `pkgs.glibcLocales` from ~7min to
slightly under 1min.

[1] https://sourceware.org/pipermail/libc-alpha/2025-October/170803.html
This commit is contained in:
Maximilian Bosch
2025-10-05 23:00:57 +02:00
parent cfb771c8a5
commit 700d7ec879
4 changed files with 98 additions and 7 deletions
@@ -0,0 +1,50 @@
From 38e9742ed794febbc8caa4177df14d097607a2d9 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 5 Oct 2025 12:45:25 +0200
Subject: [PATCH 1/2] localedata: allow reproducible & parallel install of
locale-archive
The problem with the current approach is that N processes are compiling
a locale and updating the locale-archive at the same time. Hence,
there's a race which process gets to update the archive first and thus
the build is not reproducible[1].
This patch changes that by adding the `--no-archive` flag to the
`$(LOCALEDEF)` invocation in `build-one-locale` to parallelize the build
process in a race-free manner. A single `localedef --add-to-archive`
then adds all previously built locales to the archive.
[1] https://github.com/NixOS/nixpkgs/issues/245360
---
localedata/Makefile | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/localedata/Makefile b/localedata/Makefile
index 01b32348d4..13f7d3a2c4 100644
--- a/localedata/Makefile
+++ b/localedata/Makefile
@@ -538,8 +538,13 @@ LOCALEDEF=I18NPATH=. GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
$(rtld-prefix) $(common-objpfx)locale/localedef
install-locales: install-locale-archive
+define install-all-locales
+ $(LOCALEDEF) $$flags --add-to-archive `find $(inst_complocaledir) -maxdepth 1 -mindepth 1 -type d | sort`
+endef
+
# Create and install the locale-archive file.
install-locale-archive: $(INSTALL-SUPPORTED-LOCALE-ARCHIVE)
+ $(install-all-locales)
# Create and install the locales individually (no archive).
install-locale-files: $(INSTALL-SUPPORTED-LOCALE-FILES)
@@ -565,6 +570,7 @@ define build-one-locale
$(LOCALEDEF) $$flags --alias-file=../intl/locale.alias \
-i locales/$$input -f charmaps/$$charset \
$(addprefix --prefix=,$(install_root)) $$locale \
+ --no-archive \
&& echo ' done';
endef
--
2.50.1
@@ -0,0 +1,42 @@
From 887cb6906afa2b2d8abf35ccfc33ce7b8dc902f8 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 5 Oct 2025 12:48:43 +0200
Subject: [PATCH 2/2] Makeconfig: make inst_complocaledir overridable
On NixOS, we build the locales in a separate package from glibc itself.
Also, packages aren't installed into a common location such as `/usr`,
but each package has its own unique prefix below `/nix/store`.
For us, the current state is a problem because `localedef` is built
in a different package and expects DESTDIR to be the unique
package-prefix of `glibc` itself. Our current workaround is to
invoke `localedef` with `--prefix=$TMPDIR` and copy the contents
to the correct prefix. This was doable without patching when
`localedef` was doing build & locale-archive creation at the
same time.
With the previous patch, the Makefile uses `$(inst_complocaledir)` to
tell localedef where the previously built locales are. In NixOS, we now
need to override `$(inst_complocaledir)` to point to
`$TMPDIR/$(glibc.out)`. Hence, this patch makes the variable overridable
from env.
---
Makeconfig | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Makeconfig b/Makeconfig
index 9eda4fa528..43d88325bc 100644
--- a/Makeconfig
+++ b/Makeconfig
@@ -197,7 +197,7 @@ inst_zonedir = $(install_root)$(zonedir)
ifndef complocaledir
complocaledir = $(libdir)/locale
endif
-inst_complocaledir = $(install_root)$(complocaledir)
+inst_complocaledir ?= $(install_root)$(complocaledir)
# Where to install the message catalog data files (which are
# machine-independent).
--
2.50.1
@@ -111,6 +111,10 @@ stdenv.mkDerivation (
& https://github.com/NixOS/nixpkgs/pull/188492#issuecomment-1233802991
*/
./reenable_DT_HASH.patch
# enable parallel & reproducible build of glibcLocales
./0001-localedata-allow-reproducible-parallel-install-of-lo.patch
./0002-Makeconfig-make-inst_complocaledir-overridable.patch
]
/*
NVCC does not support ARM intrinsics. Since <math.h> is pulled in by almost
+2 -7
View File
@@ -61,7 +61,8 @@
# $TMPDIR/nix/store/...-glibc-.../lib/locale/locale-archive.
LOCALEDEF_FLAGS+=" --prefix=$TMPDIR"
mkdir -p $TMPDIR/"${buildPackages.glibc.out}/lib/locale"
export inst_complocaledir="$TMPDIR/"${buildPackages.glibc.out}/lib/locale
mkdir -p "$inst_complocaledir"
echo 'C.UTF-8/UTF-8 \' >> ../glibc-2*/localedata/SUPPORTED
@@ -88,12 +89,6 @@
echo SUPPORTED-LOCALES='${toString locales}' > ../glibc-2*/localedata/SUPPORTED
'';
# Current `nixpkgs` way of building locales is not compatible with
# parallel install. `locale-archive` is updated in parallel with
# multiple `localedef` processes and causes non-deterministic result:
# https://github.com/NixOS/nixpkgs/issues/245360
enableParallelBuilding = false;
makeFlags = (previousAttrs.makeFlags or [ ]) ++ [
"localedata/install-locales"
"localedir=${placeholder "out"}/lib/locale"