nixos/etc: clear stale opaque markers from mutable overlay upperdir

Fix by walking the newly-mounted metadata layer before the overlay is
(re)mounted and removing trusted.overlay.opaque from any upperdir
directory that now has a counterpart in the lowerdir. This turns those
directories back into merged views: user-placed files and individual
whiteouts are preserved, only the blanket hiding of lowerdir content
is undone.

Applied in both the activation script (switch-to-configuration) and
the initrd rw-etc service (boot), since the upperdir persists across
reboots. Uses the new clear-etc-opaque entrypoint of nixos-init so the
initrd stays bash-free.

Fixes #505475

nixos/modules/system/etc/etc-activation.nix

@@ -1,7 +1,6 @@
 {
   config,
   lib,
-  pkgs,
   ...
 }:
 
@@ -51,6 +50,9 @@
       ];
 
       boot.initrd.systemd = {
+        storePaths = lib.mkIf config.system.etc.overlay.mutable [
+          "${config.system.nixos-init.package}/bin/clear-etc-opaque"
+        ];
         mounts = [
           {
             where = "/run/nixos-etc-metadata";
@@ -131,13 +133,20 @@
               before = [ "initrd-fs.target" ];
               unitConfig = {
                 DefaultDependencies = false;
-                RequiresMountsFor = "/sysroot";
+                RequiresMountsFor = [
+                  "/sysroot"
+                  # Needed so we can clear stale opaque markers from the
+                  # upperdir based on the contents of the new metadata layer
+                  # before the overlay is mounted.
+                  "/run/nixos-etc-metadata"
+                ];
               };
               serviceConfig = {
                 Type = "oneshot";
-                ExecStart = ''
-                  /bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work
-                '';
+                ExecStart = [
+                  "/bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work"
+                  "${config.system.nixos-init.package}/bin/clear-etc-opaque /run/nixos-etc-metadata /sysroot/.rw-etc/upper"
+                ];
               };
             };
           })

nixos/modules/system/etc/etc.nix

@@ -285,6 +285,13 @@ in
             tmpMetadataMount=$(TMPDIR="/run" mktemp --directory -t nixos-etc-metadata.XXXXXXXXXX)
             mount --type erofs --options ro,nodev,nosuid ${config.system.build.etcMetadataImage} "$tmpMetadataMount"
 
+            ${lib.optionalString config.system.etc.overlay.mutable ''
+              # Clear stale opaque markers from the upperdir so that lowerdir
+              # entries added by the new generation are not hidden.
+              # See https://github.com/NixOS/nixpkgs/issues/505475
+              ${config.system.nixos-init.package}/bin/clear-etc-opaque "$tmpMetadataMount" /.rw-etc/upper
+            ''}
+
             # There was no previous /etc mounted. This happens when we're called
             # directly without an initrd, like with nixos-enter.
             if ! mountpoint -q /etc; then

nixos/tests/activation/etc-overlay-mutable.nix

@@ -122,6 +122,10 @@
         machine.succeed("mkdir -p /.rw-etc/upper/pam.d")
         machine.succeed("setfattr -h -n trusted.overlay.opaque -v y /.rw-etc/upper/pam.d")
         machine.succeed("getfattr -h -n trusted.overlay.opaque /.rw-etc/upper/pam.d")
+        # Also create a non-opaque upperdir directory that exists in the
+        # metadata layer, to ensure clear-etc-opaque tolerates the
+        # already-clear case.
+        machine.succeed("mkdir -p /.rw-etc/upper/systemd")
 
         # Reboot and verify the initrd rw-etc service cleared the opaque marker.
         machine.shutdown()