From bbe152dcfe1c45585a312cf597fc10e6dd3c82fe Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 5 Oct 2025 13:07:29 +0200 Subject: [PATCH 1/3] glibcLocales: use substituteInPlace for Makefile fix That way we'll notice if the code changes to examine if this either needs to be updated or removed. --- pkgs/development/libraries/glibc/locales.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/development/libraries/glibc/locales.nix b/pkgs/development/libraries/glibc/locales.nix index bd61f72fc28e..8b1a4df4838f 100644 --- a/pkgs/development/libraries/glibc/locales.nix +++ b/pkgs/development/libraries/glibc/locales.nix @@ -54,7 +54,8 @@ echo 'C.UTF-8/UTF-8 \' >> ../glibc-2*/localedata/SUPPORTED # Hack to allow building of the locales (needed since glibc-2.12) - sed -i -e 's,^$(rtld-prefix) $(common-objpfx)locale/localedef,localedef $(LOCALEDEF_FLAGS),' ../glibc-2*/localedata/Makefile + substituteInPlace ../glibc-2*/localedata/Makefile \ + --replace-fail '$(rtld-prefix) $(common-objpfx)locale/localedef' 'localedef $(LOCALEDEF_FLAGS)' '' + lib.optionalString (!allLocales) '' # Check that all locales to be built are supported From cfb771c8a5a7b13e3d4f0ea48ca155d100c3e687 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 5 Oct 2025 13:14:36 +0200 Subject: [PATCH 2/3] glibcLocales: remove builder There's no particular reason to have a special builder. Fold all the changes it made into the derivation. --- .../libraries/glibc/locales-builder.sh | 15 ------------- pkgs/development/libraries/glibc/locales.nix | 22 ++++++++++++++----- 2 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 pkgs/development/libraries/glibc/locales-builder.sh diff --git a/pkgs/development/libraries/glibc/locales-builder.sh b/pkgs/development/libraries/glibc/locales-builder.sh deleted file mode 100644 index 6f10476f1089..000000000000 --- a/pkgs/development/libraries/glibc/locales-builder.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Glibc cannot have itself in its RPATH. -export NIX_NO_SELF_RPATH=1 - -postConfigure() { - # Hack: get rid of the `-static' flag set by the bootstrap stdenv. - # This has to be done *after* `configure' because it builds some - # test binaries. - export NIX_CFLAGS_LINK= - export NIX_LDFLAGS_BEFORE= - - export NIX_DONT_SET_RPATH=1 - unset CFLAGS -} - -genericBuild diff --git a/pkgs/development/libraries/glibc/locales.nix b/pkgs/development/libraries/glibc/locales.nix index 8b1a4df4838f..e8c5565ab30a 100644 --- a/pkgs/development/libraries/glibc/locales.nix +++ b/pkgs/development/libraries/glibc/locales.nix @@ -31,13 +31,25 @@ ( finalAttrs: previousAttrs: { - builder = ./locales-builder.sh; - outputs = [ "out" ]; - LOCALEDEF_FLAGS = [ - (if stdenv.hostPlatform.isLittleEndian then "--little-endian" else "--big-endian") - ]; + env = (previousAttrs.env or { }) // { + LOCALEDEF_FLAGS = if stdenv.hostPlatform.isLittleEndian then "--little-endian" else "--big-endian"; + + # Glibc cannot have itself in its RPATH. + NIX_NO_SELF_RPATH = 1; + }; + + postConfigure = (previousAttrs.postConfigure or "") + '' + # Hack: get rid of the `-static' flag set by the bootstrap stdenv. + # This has to be done *after* `configure' because it builds some + # test binaries. + export NIX_CFLAGS_LINK= + export NIX_LDFLAGS_BEFORE= + + export NIX_DONT_SET_RPATH=1 + unset CFLAGS + ''; preBuild = (previousAttrs.preBuild or "") From 700d7ec879f4bce893aa14e0ba880ed99fe1fb29 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 5 Oct 2025 13:17:27 +0200 Subject: [PATCH 3/3] glibcLocales: re-enable parallel build This was originally turned off in #245360 because the approach was non-reproducible, i.e. all N concurrent jobs were racing to update `locale-archive`. glibc's localedef allows to separate the build from the install, i.e. `localedef --no-archive` allows to only perform the build of the locale which can be parallelized again. After that, a single process adds all newly built locales into the archive. This isn't supported by the Makefiles in upstream glibc yet[1], hence the additional patches. For me, this reduces the build-time of `pkgs.glibcLocales` from ~7min to slightly under 1min. [1] https://sourceware.org/pipermail/libc-alpha/2025-October/170803.html --- ...-reproducible-parallel-install-of-lo.patch | 50 +++++++++++++++++++ ...-make-inst_complocaledir-overridable.patch | 42 ++++++++++++++++ pkgs/development/libraries/glibc/common.nix | 4 ++ pkgs/development/libraries/glibc/locales.nix | 9 +--- 4 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 pkgs/development/libraries/glibc/0001-localedata-allow-reproducible-parallel-install-of-lo.patch create mode 100644 pkgs/development/libraries/glibc/0002-Makeconfig-make-inst_complocaledir-overridable.patch diff --git a/pkgs/development/libraries/glibc/0001-localedata-allow-reproducible-parallel-install-of-lo.patch b/pkgs/development/libraries/glibc/0001-localedata-allow-reproducible-parallel-install-of-lo.patch new file mode 100644 index 000000000000..276e9cc0bf32 --- /dev/null +++ b/pkgs/development/libraries/glibc/0001-localedata-allow-reproducible-parallel-install-of-lo.patch @@ -0,0 +1,50 @@ +From 38e9742ed794febbc8caa4177df14d097607a2d9 Mon Sep 17 00:00:00 2001 +From: Maximilian Bosch +Date: Sun, 5 Oct 2025 12:45:25 +0200 +Subject: [PATCH 1/2] localedata: allow reproducible & parallel install of + locale-archive + +The problem with the current approach is that N processes are compiling +a locale and updating the locale-archive at the same time. Hence, +there's a race which process gets to update the archive first and thus +the build is not reproducible[1]. + +This patch changes that by adding the `--no-archive` flag to the +`$(LOCALEDEF)` invocation in `build-one-locale` to parallelize the build +process in a race-free manner. A single `localedef --add-to-archive` +then adds all previously built locales to the archive. + +[1] https://github.com/NixOS/nixpkgs/issues/245360 +--- + localedata/Makefile | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/localedata/Makefile b/localedata/Makefile +index 01b32348d4..13f7d3a2c4 100644 +--- a/localedata/Makefile ++++ b/localedata/Makefile +@@ -538,8 +538,13 @@ LOCALEDEF=I18NPATH=. GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \ + $(rtld-prefix) $(common-objpfx)locale/localedef + install-locales: install-locale-archive + ++define install-all-locales ++ $(LOCALEDEF) $$flags --add-to-archive `find $(inst_complocaledir) -maxdepth 1 -mindepth 1 -type d | sort` ++endef ++ + # Create and install the locale-archive file. + install-locale-archive: $(INSTALL-SUPPORTED-LOCALE-ARCHIVE) ++ $(install-all-locales) + # Create and install the locales individually (no archive). + install-locale-files: $(INSTALL-SUPPORTED-LOCALE-FILES) + +@@ -565,6 +570,7 @@ define build-one-locale + $(LOCALEDEF) $$flags --alias-file=../intl/locale.alias \ + -i locales/$$input -f charmaps/$$charset \ + $(addprefix --prefix=,$(install_root)) $$locale \ ++ --no-archive \ + && echo ' done'; + endef + +-- +2.50.1 + diff --git a/pkgs/development/libraries/glibc/0002-Makeconfig-make-inst_complocaledir-overridable.patch b/pkgs/development/libraries/glibc/0002-Makeconfig-make-inst_complocaledir-overridable.patch new file mode 100644 index 000000000000..c727912a6d84 --- /dev/null +++ b/pkgs/development/libraries/glibc/0002-Makeconfig-make-inst_complocaledir-overridable.patch @@ -0,0 +1,42 @@ +From 887cb6906afa2b2d8abf35ccfc33ce7b8dc902f8 Mon Sep 17 00:00:00 2001 +From: Maximilian Bosch +Date: Sun, 5 Oct 2025 12:48:43 +0200 +Subject: [PATCH 2/2] Makeconfig: make inst_complocaledir overridable + +On NixOS, we build the locales in a separate package from glibc itself. +Also, packages aren't installed into a common location such as `/usr`, +but each package has its own unique prefix below `/nix/store`. + +For us, the current state is a problem because `localedef` is built +in a different package and expects DESTDIR to be the unique +package-prefix of `glibc` itself. Our current workaround is to +invoke `localedef` with `--prefix=$TMPDIR` and copy the contents +to the correct prefix. This was doable without patching when +`localedef` was doing build & locale-archive creation at the +same time. + +With the previous patch, the Makefile uses `$(inst_complocaledir)` to +tell localedef where the previously built locales are. In NixOS, we now +need to override `$(inst_complocaledir)` to point to +`$TMPDIR/$(glibc.out)`. Hence, this patch makes the variable overridable +from env. +--- + Makeconfig | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makeconfig b/Makeconfig +index 9eda4fa528..43d88325bc 100644 +--- a/Makeconfig ++++ b/Makeconfig +@@ -197,7 +197,7 @@ inst_zonedir = $(install_root)$(zonedir) + ifndef complocaledir + complocaledir = $(libdir)/locale + endif +-inst_complocaledir = $(install_root)$(complocaledir) ++inst_complocaledir ?= $(install_root)$(complocaledir) + + # Where to install the message catalog data files (which are + # machine-independent). +-- +2.50.1 + diff --git a/pkgs/development/libraries/glibc/common.nix b/pkgs/development/libraries/glibc/common.nix index a35e5085f87c..5293c6579c5d 100644 --- a/pkgs/development/libraries/glibc/common.nix +++ b/pkgs/development/libraries/glibc/common.nix @@ -111,6 +111,10 @@ stdenv.mkDerivation ( & https://github.com/NixOS/nixpkgs/pull/188492#issuecomment-1233802991 */ ./reenable_DT_HASH.patch + + # enable parallel & reproducible build of glibcLocales + ./0001-localedata-allow-reproducible-parallel-install-of-lo.patch + ./0002-Makeconfig-make-inst_complocaledir-overridable.patch ] /* NVCC does not support ARM intrinsics. Since is pulled in by almost diff --git a/pkgs/development/libraries/glibc/locales.nix b/pkgs/development/libraries/glibc/locales.nix index e8c5565ab30a..98f04f22dbcd 100644 --- a/pkgs/development/libraries/glibc/locales.nix +++ b/pkgs/development/libraries/glibc/locales.nix @@ -61,7 +61,8 @@ # $TMPDIR/nix/store/...-glibc-.../lib/locale/locale-archive. LOCALEDEF_FLAGS+=" --prefix=$TMPDIR" - mkdir -p $TMPDIR/"${buildPackages.glibc.out}/lib/locale" + export inst_complocaledir="$TMPDIR/"${buildPackages.glibc.out}/lib/locale + mkdir -p "$inst_complocaledir" echo 'C.UTF-8/UTF-8 \' >> ../glibc-2*/localedata/SUPPORTED @@ -88,12 +89,6 @@ echo SUPPORTED-LOCALES='${toString locales}' > ../glibc-2*/localedata/SUPPORTED ''; - # Current `nixpkgs` way of building locales is not compatible with - # parallel install. `locale-archive` is updated in parallel with - # multiple `localedef` processes and causes non-deterministic result: - # https://github.com/NixOS/nixpkgs/issues/245360 - enableParallelBuilding = false; - makeFlags = (previousAttrs.makeFlags or [ ]) ++ [ "localedata/install-locales" "localedir=${placeholder "out"}/lib/locale"