From 3c5db5ceba893467cee119845b10588fd8309b0b Mon Sep 17 00:00:00 2001 From: r-vdp Date: Sun, 19 Apr 2026 22:33:18 +0200 Subject: [PATCH] nixos-init: add clear-etc-opaque entrypoint The mutable /etc overlay can accumulate stale trusted.overlay.opaque xattrs in its upperdir that hide lowerdir entries added by later generations (see NixOS/nixpkgs#505475). The fixup needs to run in initrd before the overlay is mounted, so we don't want to use bash for this. Add a small clear-etc-opaque entrypoint to the nixos-init binary that walks the metadata layer and removes the opaque xattr from matching upperdir directories. This adds the xattr crate and one extra symlink to the initrd. --- pkgs/by-name/ni/nixos-init/Cargo.lock | 11 ++ pkgs/by-name/ni/nixos-init/Cargo.toml | 1 + pkgs/by-name/ni/nixos-init/README.md | 3 + pkgs/by-name/ni/nixos-init/package.nix | 1 + pkgs/by-name/ni/nixos-init/src/etc_overlay.rs | 170 ++++++++++++++++++ pkgs/by-name/ni/nixos-init/src/lib.rs | 2 + pkgs/by-name/ni/nixos-init/src/main.rs | 3 +- 7 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 pkgs/by-name/ni/nixos-init/src/etc_overlay.rs diff --git a/pkgs/by-name/ni/nixos-init/Cargo.lock b/pkgs/by-name/ni/nixos-init/Cargo.lock index 5aee8eedf701..e9d0c98cb115 100644 --- a/pkgs/by-name/ni/nixos-init/Cargo.lock +++ b/pkgs/by-name/ni/nixos-init/Cargo.lock @@ -142,6 +142,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "xattr", ] [[package]] @@ -506,3 +507,13 @@ checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags", ] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] diff --git a/pkgs/by-name/ni/nixos-init/Cargo.toml b/pkgs/by-name/ni/nixos-init/Cargo.toml index 46102be28304..6955ab5f091b 100644 --- a/pkgs/by-name/ni/nixos-init/Cargo.toml +++ b/pkgs/by-name/ni/nixos-init/Cargo.toml @@ -11,6 +11,7 @@ pathrs = "0.2.2" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" bootspec = "2.0.0" +xattr = "1.6.1" [dev-dependencies] tempfile = "3.20.0" diff --git a/pkgs/by-name/ni/nixos-init/README.md b/pkgs/by-name/ni/nixos-init/README.md index 4d1677db0015..96b50f3449ce 100644 --- a/pkgs/by-name/ni/nixos-init/README.md +++ b/pkgs/by-name/ni/nixos-init/README.md @@ -52,6 +52,9 @@ closure. Currently nixos-init comes in at ~500 KiB. - `find-etc`: Finds the `/etc` paths in `/sysroot` so that the initrd doesn't directly depend on the toplevel, reducing the need to rebuild the initrd on every generation. +- `clear-etc-opaque`: Clears stale `trusted.overlay.opaque` xattrs from the + mutable `/etc` overlay's upperdir before it is mounted, so that lowerdir + entries added by a new generation are not hidden. - `resolve-in-root`: Figures out the canonical path inside a chroot. ## Future diff --git a/pkgs/by-name/ni/nixos-init/package.nix b/pkgs/by-name/ni/nixos-init/package.nix index c84c711c7599..85d394e10485 100644 --- a/pkgs/by-name/ni/nixos-init/package.nix +++ b/pkgs/by-name/ni/nixos-init/package.nix @@ -47,6 +47,7 @@ rustPlatform.buildRustPackage (finalAttrs: { binaries = [ "initrd-init" "find-etc" + "clear-etc-opaque" "resolve-in-root" "env-generator" ]; diff --git a/pkgs/by-name/ni/nixos-init/src/etc_overlay.rs b/pkgs/by-name/ni/nixos-init/src/etc_overlay.rs new file mode 100644 index 000000000000..a5c681ff0869 --- /dev/null +++ b/pkgs/by-name/ni/nixos-init/src/etc_overlay.rs @@ -0,0 +1,170 @@ +use std::{ + env, fs, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, bail}; + +const OVERLAY_OPAQUE_XATTR: &str = "trusted.overlay.opaque"; + +/// Entrypoint for the `clear-etc-opaque` binary. +/// +/// When a directory is created in the mutable `/etc` overlay that does not yet +/// exist in the lowerdir, overlayfs marks it opaque in the upperdir. This is +/// correct at creation time, but becomes stale when a later generation adds +/// entries under that same directory to the metadata layer: the opaque marker +/// hides them. +/// +/// This walks the (newly mounted) metadata layer and removes +/// `trusted.overlay.opaque` from any upperdir directory that now has a +/// directory counterpart in the lowerdir, turning it back into a merged view. +/// Files the user placed in the upperdir remain visible (upperdir wins +/// per-entry) and individual whiteouts are preserved; only the blanket hiding +/// of lowerdir content is undone. +/// +/// See . +/// +/// Usage: `clear-etc-opaque ` +pub fn clear_etc_opaque() -> Result<()> { + let args: Vec = env::args().collect(); + + if args.len() != 3 { + bail!("Usage: {} ", args[0]); + } + + let metadata_mount = PathBuf::from(&args[1]); + let upperdir = PathBuf::from(&args[2]); + + if !upperdir.is_dir() { + // Nothing to clear (e.g. first boot before the upperdir is created). + log::info!( + "Upperdir {} does not exist, nothing to clear.", + upperdir.display() + ); + return Ok(()); + } + + clear_opaque_markers(&metadata_mount, &metadata_mount, &upperdir) +} + +/// Recursively walk `current` (a subtree of `metadata_root`) and clear the +/// opaque xattr from the corresponding directory in `upperdir`. +fn clear_opaque_markers(metadata_root: &Path, current: &Path, upperdir: &Path) -> Result<()> { + let entries = fs::read_dir(current) + .with_context(|| format!("Failed to read directory {}", current.display()))?; + + for entry in entries { + let entry = + entry.with_context(|| format!("Failed to read entry in {}", current.display()))?; + + // Use the entry's own type info (no symlink following) so we only + // recurse into real directories of the metadata image. + if !entry + .file_type() + .with_context(|| format!("Failed to stat {}", entry.path().display()))? + .is_dir() + { + continue; + } + + let path = entry.path(); + let rel = path + .strip_prefix(metadata_root) + .context("Failed to strip metadata root prefix")?; + let target = upperdir.join(rel); + + // Only act on real directories in the upperdir; an opaque marker on a + // non-directory would be meaningless and we must not follow symlinks + // out of the upperdir. + match fs::symlink_metadata(&target) { + Ok(meta) if meta.is_dir() => { + remove_opaque_xattr(&target); + // Only recurse when the upperdir also has this directory: + // deeper lowerdir directories without an upperdir counterpart + // cannot carry stale markers. + clear_opaque_markers(metadata_root, &path, upperdir)?; + } + // Missing or not a directory: nothing to do for this subtree. + _ => {} + } + } + + Ok(()) +} + +/// Remove the `trusted.overlay.opaque` xattr from `path` if present. +fn remove_opaque_xattr(path: &Path) { + // Check first instead of removing unconditionally: lremovexattr(2) reports + // a missing attribute as ENODATA, which std does not map to a stable + // io::ErrorKind, so distinguishing it from real errors is awkward. + match xattr::get(path, OVERLAY_OPAQUE_XATTR) { + Ok(None) => return, + Ok(Some(_)) => {} + Err(err) => { + log::warn!( + "Failed to read {OVERLAY_OPAQUE_XATTR} on {}: {err}.", + path.display() + ); + return; + } + } + + match xattr::remove(path, OVERLAY_OPAQUE_XATTR) { + Ok(()) => { + log::info!("Cleared stale opaque marker from {}.", path.display()); + } + Err(err) => { + // Don't abort the boot over this; the worst case is that some + // declaratively-managed /etc entries stay hidden, which is what + // would happen anyway without this fixup. + log::warn!( + "Failed to remove {OVERLAY_OPAQUE_XATTR} from {}: {err}.", + path.display() + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use tempfile::tempdir; + + #[test] + fn clears_opaque_only_for_matching_dirs() -> Result<()> { + if !xattr::SUPPORTED_PLATFORM { + return Ok(()); + } + + let metadata = tempdir()?; + let upper = tempdir()?; + + // lowerdir gained nixos/sub in the new generation. + fs::create_dir_all(metadata.path().join("nixos/sub"))?; + // upperdir has an opaque nixos/ from before. + fs::create_dir_all(upper.path().join("nixos"))?; + // upperdir directory without a lowerdir counterpart: we only clear + // markers where the lowerdir has a matching directory, so this one + // must stay opaque. + fs::create_dir_all(upper.path().join("only-upper"))?; + + // The build sandbox usually lacks CAP_SYS_ADMIN, so trusted.* xattrs + // cannot be set. Skip in that case rather than fail the build. + if xattr::set(upper.path().join("nixos"), OVERLAY_OPAQUE_XATTR, b"y").is_err() { + eprintln!("skipping: cannot set trusted.* xattrs in this environment"); + return Ok(()); + } + xattr::set(upper.path().join("only-upper"), OVERLAY_OPAQUE_XATTR, b"y")?; + + clear_opaque_markers(metadata.path(), metadata.path(), upper.path())?; + + assert!(xattr::get(upper.path().join("nixos"), OVERLAY_OPAQUE_XATTR)?.is_none()); + assert_eq!( + xattr::get(upper.path().join("only-upper"), OVERLAY_OPAQUE_XATTR)?.as_deref(), + Some(b"y".as_slice()) + ); + + Ok(()) + } +} diff --git a/pkgs/by-name/ni/nixos-init/src/lib.rs b/pkgs/by-name/ni/nixos-init/src/lib.rs index e9a716724051..c781757c9b82 100644 --- a/pkgs/by-name/ni/nixos-init/src/lib.rs +++ b/pkgs/by-name/ni/nixos-init/src/lib.rs @@ -1,6 +1,7 @@ mod activate; mod config; mod env_generator; +mod etc_overlay; mod find_etc; mod fs; mod init; @@ -16,6 +17,7 @@ use anyhow::{Context, Result, bail}; pub use crate::{ activate::activate, env_generator::env_generator, + etc_overlay::clear_etc_opaque, find_etc::find_etc, init::init, initrd_init::initrd_init, diff --git a/pkgs/by-name/ni/nixos-init/src/main.rs b/pkgs/by-name/ni/nixos-init/src/main.rs index 4f74415b8d36..bef10017f1ea 100644 --- a/pkgs/by-name/ni/nixos-init/src/main.rs +++ b/pkgs/by-name/ni/nixos-init/src/main.rs @@ -2,7 +2,7 @@ use std::{env, io::Write, process::ExitCode}; use log::Level; -use nixos_init::{env_generator, find_etc, initrd_init, resolve_in_root}; +use nixos_init::{clear_etc_opaque, env_generator, find_etc, initrd_init, resolve_in_root}; fn main() -> ExitCode { let arg0 = env::args() @@ -12,6 +12,7 @@ fn main() -> ExitCode { setup_logger(); let entrypoint = match arg0.as_str() { + "clear-etc-opaque" => clear_etc_opaque, "find-etc" => find_etc, "resolve-in-root" => resolve_in_root, "initrd-init" => initrd_init,