glibcLocales: re-enable parallelism, refactor (#448829)
This commit is contained in:
Arnout Engelen
+50
@@ -0,0 +1,50 @@
|
||||
From 38e9742ed794febbc8caa4177df14d097607a2d9 Mon Sep 17 00:00:00 2001
|
||||
From: Maximilian Bosch <maximilian@mbosch.me>
|
||||
Date: Sun, 5 Oct 2025 12:45:25 +0200
|
||||
Subject: [PATCH 1/2] localedata: allow reproducible & parallel install of
|
||||
locale-archive
|
||||
|
||||
The problem with the current approach is that N processes are compiling
|
||||
a locale and updating the locale-archive at the same time. Hence,
|
||||
there's a race which process gets to update the archive first and thus
|
||||
the build is not reproducible[1].
|
||||
|
||||
This patch changes that by adding the `--no-archive` flag to the
|
||||
`$(LOCALEDEF)` invocation in `build-one-locale` to parallelize the build
|
||||
process in a race-free manner. A single `localedef --add-to-archive`
|
||||
then adds all previously built locales to the archive.
|
||||
|
||||
[1] https://github.com/NixOS/nixpkgs/issues/245360
|
||||
---
|
||||
localedata/Makefile | 6 ++++++
|
||||
1 file changed, 6 insertions(+)
|
||||
|
||||
diff --git a/localedata/Makefile b/localedata/Makefile
|
||||
index 01b32348d4..13f7d3a2c4 100644
|
||||
--- a/localedata/Makefile
|
||||
+++ b/localedata/Makefile
|
||||
@@ -538,8 +538,13 @@ LOCALEDEF=I18NPATH=. GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
|
||||
$(rtld-prefix) $(common-objpfx)locale/localedef
|
||||
install-locales: install-locale-archive
|
||||
|
||||
+define install-all-locales
|
||||
+ $(LOCALEDEF) $$flags --add-to-archive `find $(inst_complocaledir) -maxdepth 1 -mindepth 1 -type d | sort`
|
||||
+endef
|
||||
+
|
||||
# Create and install the locale-archive file.
|
||||
install-locale-archive: $(INSTALL-SUPPORTED-LOCALE-ARCHIVE)
|
||||
+ $(install-all-locales)
|
||||
# Create and install the locales individually (no archive).
|
||||
install-locale-files: $(INSTALL-SUPPORTED-LOCALE-FILES)
|
||||
|
||||
@@ -565,6 +570,7 @@ define build-one-locale
|
||||
$(LOCALEDEF) $$flags --alias-file=../intl/locale.alias \
|
||||
-i locales/$$input -f charmaps/$$charset \
|
||||
$(addprefix --prefix=,$(install_root)) $$locale \
|
||||
+ --no-archive \
|
||||
&& echo ' done';
|
||||
endef
|
||||
|
||||
--
|
||||
2.50.1
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
From 887cb6906afa2b2d8abf35ccfc33ce7b8dc902f8 Mon Sep 17 00:00:00 2001
|
||||
From: Maximilian Bosch <maximilian@mbosch.me>
|
||||
Date: Sun, 5 Oct 2025 12:48:43 +0200
|
||||
Subject: [PATCH 2/2] Makeconfig: make inst_complocaledir overridable
|
||||
|
||||
On NixOS, we build the locales in a separate package from glibc itself.
|
||||
Also, packages aren't installed into a common location such as `/usr`,
|
||||
but each package has its own unique prefix below `/nix/store`.
|
||||
|
||||
For us, the current state is a problem because `localedef` is built
|
||||
in a different package and expects DESTDIR to be the unique
|
||||
package-prefix of `glibc` itself. Our current workaround is to
|
||||
invoke `localedef` with `--prefix=$TMPDIR` and copy the contents
|
||||
to the correct prefix. This was doable without patching when
|
||||
`localedef` was doing build & locale-archive creation at the
|
||||
same time.
|
||||
|
||||
With the previous patch, the Makefile uses `$(inst_complocaledir)` to
|
||||
tell localedef where the previously built locales are. In NixOS, we now
|
||||
need to override `$(inst_complocaledir)` to point to
|
||||
`$TMPDIR/$(glibc.out)`. Hence, this patch makes the variable overridable
|
||||
from env.
|
||||
---
|
||||
Makeconfig | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Makeconfig b/Makeconfig
|
||||
index 9eda4fa528..43d88325bc 100644
|
||||
--- a/Makeconfig
|
||||
+++ b/Makeconfig
|
||||
@@ -197,7 +197,7 @@ inst_zonedir = $(install_root)$(zonedir)
|
||||
ifndef complocaledir
|
||||
complocaledir = $(libdir)/locale
|
||||
endif
|
||||
-inst_complocaledir = $(install_root)$(complocaledir)
|
||||
+inst_complocaledir ?= $(install_root)$(complocaledir)
|
||||
|
||||
# Where to install the message catalog data files (which are
|
||||
# machine-independent).
|
||||
--
|
||||
2.50.1
|
||||
|
||||
@@ -111,6 +111,10 @@ stdenv.mkDerivation (
|
||||
& https://github.com/NixOS/nixpkgs/pull/188492#issuecomment-1233802991
|
||||
*/
|
||||
./reenable_DT_HASH.patch
|
||||
|
||||
# enable parallel & reproducible build of glibcLocales
|
||||
./0001-localedata-allow-reproducible-parallel-install-of-lo.patch
|
||||
./0002-Makeconfig-make-inst_complocaledir-overridable.patch
|
||||
]
|
||||
/*
|
||||
NVCC does not support ARM intrinsics. Since <math.h> is pulled in by almost
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
# Glibc cannot have itself in its RPATH.
|
||||
export NIX_NO_SELF_RPATH=1
|
||||
|
||||
postConfigure() {
|
||||
# Hack: get rid of the `-static' flag set by the bootstrap stdenv.
|
||||
# This has to be done *after* `configure' because it builds some
|
||||
# test binaries.
|
||||
export NIX_CFLAGS_LINK=
|
||||
export NIX_LDFLAGS_BEFORE=
|
||||
|
||||
export NIX_DONT_SET_RPATH=1
|
||||
unset CFLAGS
|
||||
}
|
||||
|
||||
genericBuild
|
||||
@@ -31,13 +31,25 @@
|
||||
(
|
||||
finalAttrs: previousAttrs: {
|
||||
|
||||
builder = ./locales-builder.sh;
|
||||
|
||||
outputs = [ "out" ];
|
||||
|
||||
LOCALEDEF_FLAGS = [
|
||||
(if stdenv.hostPlatform.isLittleEndian then "--little-endian" else "--big-endian")
|
||||
];
|
||||
env = (previousAttrs.env or { }) // {
|
||||
LOCALEDEF_FLAGS = if stdenv.hostPlatform.isLittleEndian then "--little-endian" else "--big-endian";
|
||||
|
||||
# Glibc cannot have itself in its RPATH.
|
||||
NIX_NO_SELF_RPATH = 1;
|
||||
};
|
||||
|
||||
postConfigure = (previousAttrs.postConfigure or "") + ''
|
||||
# Hack: get rid of the `-static' flag set by the bootstrap stdenv.
|
||||
# This has to be done *after* `configure' because it builds some
|
||||
# test binaries.
|
||||
export NIX_CFLAGS_LINK=
|
||||
export NIX_LDFLAGS_BEFORE=
|
||||
|
||||
export NIX_DONT_SET_RPATH=1
|
||||
unset CFLAGS
|
||||
'';
|
||||
|
||||
preBuild =
|
||||
(previousAttrs.preBuild or "")
|
||||
@@ -49,12 +61,14 @@
|
||||
# $TMPDIR/nix/store/...-glibc-.../lib/locale/locale-archive.
|
||||
LOCALEDEF_FLAGS+=" --prefix=$TMPDIR"
|
||||
|
||||
mkdir -p $TMPDIR/"${buildPackages.glibc.out}/lib/locale"
|
||||
export inst_complocaledir="$TMPDIR/"${buildPackages.glibc.out}/lib/locale
|
||||
mkdir -p "$inst_complocaledir"
|
||||
|
||||
echo 'C.UTF-8/UTF-8 \' >> ../glibc-2*/localedata/SUPPORTED
|
||||
|
||||
# Hack to allow building of the locales (needed since glibc-2.12)
|
||||
sed -i -e 's,^$(rtld-prefix) $(common-objpfx)locale/localedef,localedef $(LOCALEDEF_FLAGS),' ../glibc-2*/localedata/Makefile
|
||||
substituteInPlace ../glibc-2*/localedata/Makefile \
|
||||
--replace-fail '$(rtld-prefix) $(common-objpfx)locale/localedef' 'localedef $(LOCALEDEF_FLAGS)'
|
||||
''
|
||||
+ lib.optionalString (!allLocales) ''
|
||||
# Check that all locales to be built are supported
|
||||
@@ -75,12 +89,6 @@
|
||||
echo SUPPORTED-LOCALES='${toString locales}' > ../glibc-2*/localedata/SUPPORTED
|
||||
'';
|
||||
|
||||
# Current `nixpkgs` way of building locales is not compatible with
|
||||
# parallel install. `locale-archive` is updated in parallel with
|
||||
# multiple `localedef` processes and causes non-deterministic result:
|
||||
# https://github.com/NixOS/nixpkgs/issues/245360
|
||||
enableParallelBuilding = false;
|
||||
|
||||
makeFlags = (previousAttrs.makeFlags or [ ]) ++ [
|
||||
"localedata/install-locales"
|
||||
"localedir=${placeholder "out"}/lib/locale"
|
||||
|
||||
Reference in New Issue
Block a user