diff --git a/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml
index 930f6b2066e3..da765edcd08e 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml
@@ -30,7 +30,7 @@
Backward Incompatibilities
-
+
carnix and cratesIO has
@@ -42,6 +42,40 @@
instead.
+
+
+ The EC2 image module no longer fetches instance metadata in
+ stage-1. This results in a significantly smaller initramfs,
+ since network drivers no longer need to be included, and
+ faster boots, since metadata fetching can happen in parallel
+ with startup of other services. This breaks services which
+ rely on metadata being present by the time stage-2 is entered.
+ Anything which reads EC2 metadata from
+ /etc/ec2-metadata should now have an
+ after dependency on
+ fetch-ec2-metadata.service
+
+
+
+
+ The EC2 image module previously detected and automatically
+ mounted ext3-formatted instance store devices and partitions
+ in stage-1 (initramfs), storing /tmp on the
+ first discovered device. This behaviour, which only catered to
+ very specific use cases and could not be disabled, has been
+ removed. Users relying on this should provide their own
+ implementation, and probably use ext4 and perform the mount in
+ stage-2.
+
+
+
+
+ The EC2 image module previously detected and activated
+ swap-formatted instance store devices and partitions in
+ stage-1 (initramfs). This behaviour has been removed. Users
+ relying on this should provide their own implementation.
+
+
diff --git a/nixos/doc/manual/release-notes/rl-2305.section.md b/nixos/doc/manual/release-notes/rl-2305.section.md
index 0e8e1ad0b230..3cbf4621a787 100644
--- a/nixos/doc/manual/release-notes/rl-2305.section.md
+++ b/nixos/doc/manual/release-notes/rl-2305.section.md
@@ -22,6 +22,13 @@ In addition to numerous new and upgraded packages, this release has the followin
- `carnix` and `cratesIO` has been removed due to being unmaintained, use alternatives such as [naersk](https://github.com/nix-community/naersk) and [crate2nix](https://github.com/kolloch/crate2nix) instead.
+- The EC2 image module no longer fetches instance metadata in stage-1. This results in a significantly smaller initramfs, since network drivers no longer need to be included, and faster boots, since metadata fetching can happen in parallel with startup of other services.
+ This breaks services which rely on metadata being present by the time stage-2 is entered. Anything which reads EC2 metadata from `/etc/ec2-metadata` should now have an `after` dependency on `fetch-ec2-metadata.service`
+
+- The EC2 image module previously detected and automatically mounted ext3-formatted instance store devices and partitions in stage-1 (initramfs), storing `/tmp` on the first discovered device. This behaviour, which only catered to very specific use cases and could not be disabled, has been removed. Users relying on this should provide their own implementation, and probably use ext4 and perform the mount in stage-2.
+
+- The EC2 image module previously detected and activated swap-formatted instance store devices and partitions in stage-1 (initramfs). This behaviour has been removed. Users relying on this should provide their own implementation.
+
## Other Notable Changes {#sec-release-23.05-notable-changes}
diff --git a/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixos/maintainers/scripts/ec2/amazon-image.nix
index e2a05a09d0c2..0db0f4d0dcca 100644
--- a/nixos/maintainers/scripts/ec2/amazon-image.nix
+++ b/nixos/maintainers/scripts/ec2/amazon-image.nix
@@ -43,7 +43,7 @@ in {
sizeMB = mkOption {
type = with types; either (enum [ "auto" ]) int;
- default = if config.ec2.hvm then 2048 else 8192;
+ default = 2048;
example = 8192;
description = lib.mdDoc "The size in MB of the image";
};
@@ -60,9 +60,6 @@ in {
''
{ modulesPath, ... }: {
imports = [ "''${modulesPath}/virtualisation/amazon-image.nix" ];
- ${optionalString config.ec2.hvm ''
- ec2.hvm = true;
- ''}
${optionalString config.ec2.efi ''
ec2.efi = true;
''}
@@ -129,9 +126,7 @@ in {
pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
fsType = "ext4";
- partitionTableType = if config.ec2.efi then "efi"
- else if config.ec2.hvm then "legacy+gpt"
- else "none";
+ partitionTableType = if config.ec2.efi then "efi" else "legacy+gpt";
diskSize = cfg.sizeMB;
diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix
index 12fe6fa44479..9751f5755f96 100644
--- a/nixos/modules/virtualisation/amazon-image.nix
+++ b/nixos/modules/virtualisation/amazon-image.nix
@@ -10,11 +10,6 @@ with lib;
let
cfg = config.ec2;
- metadataFetcher = import ./ec2-metadata-fetcher.nix {
- inherit (pkgs) curl;
- targetRoot = "$targetRoot/";
- wgetExtraOptions = "-q";
- };
in
{
@@ -31,18 +26,12 @@ in
config = {
assertions = [
- { assertion = cfg.hvm;
- message = "Paravirtualized EC2 instances are no longer supported.";
- }
- { assertion = cfg.efi -> cfg.hvm;
- message = "EC2 instances using EFI must be HVM instances.";
- }
{ assertion = versionOlder config.boot.kernelPackages.kernel.version "5.17";
message = "ENA driver fails to build with kernel >= 5.17";
}
];
- boot.growPartition = cfg.hvm;
+ boot.growPartition = true;
fileSystems."/" = mkIf (!cfg.zfs.enable) {
device = "/dev/disk/by-label/nixos";
@@ -64,9 +53,9 @@ in
boot.extraModulePackages = [
config.boot.kernelPackages.ena
];
- boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ];
- boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ];
- boot.kernelParams = mkIf cfg.hvm [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];
+ boot.initrd.kernelModules = [ "xen-blkfront" ];
+ boot.initrd.availableKernelModules = [ "nvme" ];
+ boot.kernelParams = [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];
# Prevent the nouveau kernel module from being loaded, as it
# interferes with the nvidia/nvidia-uvm modules needed for CUDA.
@@ -74,10 +63,7 @@ in
# boot.
boot.blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
- # Generate a GRUB menu. Amazon's pv-grub uses this to boot our kernel/initrd.
- boot.loader.grub.version = if cfg.hvm then 2 else 1;
- boot.loader.grub.device = if (cfg.hvm && !cfg.efi) then "/dev/xvda" else "nodev";
- boot.loader.grub.extraPerEntryConfig = mkIf (!cfg.hvm) "root (hd0)";
+ boot.loader.grub.device = if cfg.efi then "nodev" else "/dev/xvda";
boot.loader.grub.efiSupport = cfg.efi;
boot.loader.grub.efiInstallAsRemovable = cfg.efi;
boot.loader.timeout = 1;
@@ -87,67 +73,14 @@ in
terminal_input console serial
'';
- boot.initrd.network.enable = true;
-
- # Mount all formatted ephemeral disks and activate all swap devices.
- # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
- # because the set of devices is dependent on the instance type
- # (e.g. "m1.small" has one ephemeral filesystem and one swap device,
- # while "m1.large" has two ephemeral filesystems and no swap
- # devices). Also, put /tmp and /var on /disk0, since it has a lot
- # more space than the root device. Similarly, "move" /nix to /disk0
- # by layering a unionfs-fuse mount on top of it so we have a lot more space for
- # Nix operations.
- boot.initrd.postMountCommands =
- ''
- ${metadataFetcher}
-
- diskNr=0
- diskForUnionfs=
- for device in /dev/xvd[abcde]*; do
- if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
- fsType=$(blkid -o value -s TYPE "$device" || true)
- if [ "$fsType" = swap ]; then
- echo "activating swap device $device..."
- swapon "$device" || true
- elif [ "$fsType" = ext3 ]; then
- mp="/disk$diskNr"
- diskNr=$((diskNr + 1))
- if mountFS "$device" "$mp" "" ext3; then
- if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
- fi
- else
- echo "skipping unknown device type $device"
- fi
- done
-
- if [ -n "$diskForUnionfs" ]; then
- mkdir -m 755 -p $targetRoot/$diskForUnionfs/root
-
- mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
- mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
-
- if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
- mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
- mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var
-
- mkdir -p /unionfs-chroot/ro-nix
- mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
-
- mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
- mkdir -p /unionfs-chroot/rw-nix
- mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix
-
- unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
- fi
- fi
- '';
-
- boot.initrd.extraUtilsCommands =
- ''
- # We need swapon in the initrd.
- copy_bin_and_libs ${pkgs.util-linux}/sbin/swapon
- '';
+ systemd.services.fetch-ec2-metadata = {
+ wantedBy = [ "multi-user.target" ];
+ after = ["network-online.target"];
+ path = [ pkgs.curl ];
+ script = builtins.readFile ./ec2-metadata-fetcher.sh;
+ serviceConfig.Type = "oneshot";
+ serviceConfig.StandardOutput = "journal+console";
+ };
# Allow root logins only using the SSH key that the user specified
# at instance creation time.
@@ -166,8 +99,6 @@ in
# Always include cryptsetup so that Charon can use it.
environment.systemPackages = [ pkgs.cryptsetup ];
- boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
-
# EC2 has its own NTP server provided by the hypervisor
networking.timeServers = [ "169.254.169.123" ];
diff --git a/nixos/modules/virtualisation/amazon-options.nix b/nixos/modules/virtualisation/amazon-options.nix
index 227f3e433c10..915bbf9763db 100644
--- a/nixos/modules/virtualisation/amazon-options.nix
+++ b/nixos/modules/virtualisation/amazon-options.nix
@@ -2,6 +2,9 @@
let
inherit (lib) literalExpression types;
in {
+ imports = [
+ (lib.mkRemovedOptionModule [ "ec2" "hvm" ] "Only HVM instances are supported, so specifying it is no longer necessary.")
+ ];
options = {
ec2 = {
zfs = {
@@ -41,13 +44,6 @@ in {
});
};
};
- hvm = lib.mkOption {
- default = lib.versionAtLeast config.system.stateVersion "17.03";
- internal = true;
- description = lib.mdDoc ''
- Whether the EC2 instance is a HVM instance.
- '';
- };
efi = lib.mkOption {
default = pkgs.stdenv.hostPlatform.isAarch64;
defaultText = literalExpression "pkgs.stdenv.hostPlatform.isAarch64";
diff --git a/nixos/modules/virtualisation/ec2-data.nix b/nixos/modules/virtualisation/ec2-data.nix
index 1b764e7e4d80..0cc6d9938e22 100644
--- a/nixos/modules/virtualisation/ec2-data.nix
+++ b/nixos/modules/virtualisation/ec2-data.nix
@@ -18,6 +18,7 @@ with lib;
wantedBy = [ "multi-user.target" "sshd.service" ];
before = [ "sshd.service" ];
+ after = ["fetch-ec2-metadata.service"];
path = [ pkgs.iproute2 ];
diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix
deleted file mode 100644
index 760f024f33fb..000000000000
--- a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix
+++ /dev/null
@@ -1,77 +0,0 @@
-{ curl, targetRoot, wgetExtraOptions }:
-# Note: be very cautious about dependencies, each dependency grows
-# the closure of the initrd. Ideally we would not even require curl,
-# but there is no reasonable way to send an HTTP PUT request without
-# it. Note: do not be fooled: the wget referenced in this script
-# is busybox's wget, not the fully featured one with --method support.
-#
-# Make sure that every package you depend on here is already listed as
-# a channel blocker for both the full-sized and small channels.
-# Otherwise, we risk breaking user deploys in released channels.
-#
-# Also note: OpenStack's metadata service for its instances aims to be
-# compatible with the EC2 IMDS. Where possible, try to keep the set of
-# fetched metadata in sync with ./openstack-metadata-fetcher.nix .
-''
- metaDir=${targetRoot}etc/ec2-metadata
- mkdir -m 0755 -p "$metaDir"
- rm -f "$metaDir/*"
-
- get_imds_token() {
- # retry-delay of 1 selected to give the system a second to get going,
- # but not add a lot to the bootup time
- ${curl}/bin/curl \
- -v \
- --retry 3 \
- --retry-delay 1 \
- --fail \
- -X PUT \
- --connect-timeout 1 \
- -H "X-aws-ec2-metadata-token-ttl-seconds: 600" \
- http://169.254.169.254/latest/api/token
- }
-
- preflight_imds_token() {
- # retry-delay of 1 selected to give the system a second to get going,
- # but not add a lot to the bootup time
- ${curl}/bin/curl \
- -v \
- --retry 3 \
- --retry-delay 1 \
- --fail \
- --connect-timeout 1 \
- -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" \
- http://169.254.169.254/1.0/meta-data/instance-id
- }
-
- try=1
- while [ $try -le 3 ]; do
- echo "(attempt $try/3) getting an EC2 instance metadata service v2 token..."
- IMDS_TOKEN=$(get_imds_token) && break
- try=$((try + 1))
- sleep 1
- done
-
- if [ "x$IMDS_TOKEN" == "x" ]; then
- echo "failed to fetch an IMDS2v token."
- fi
-
- try=1
- while [ $try -le 10 ]; do
- echo "(attempt $try/10) validating the EC2 instance metadata service v2 token..."
- preflight_imds_token && break
- try=$((try + 1))
- sleep 1
- done
-
- echo "getting EC2 instance metadata..."
-
- wget_imds() {
- wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@";
- }
-
- wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
- (umask 077 && wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data)
- wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
- wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
-''
diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.sh b/nixos/modules/virtualisation/ec2-metadata-fetcher.sh
new file mode 100644
index 000000000000..9e204d45dbd8
--- /dev/null
+++ b/nixos/modules/virtualisation/ec2-metadata-fetcher.sh
@@ -0,0 +1,67 @@
+metaDir=/etc/ec2-metadata
+mkdir -m 0755 -p "$metaDir"
+rm -f "$metaDir/*"
+
+get_imds_token() {
+ # retry-delay of 1 selected to give the system a second to get going,
+ # but not add a lot to the bootup time
+ curl \
+ --silent \
+ --show-error \
+ --retry 3 \
+ --retry-delay 1 \
+ --fail \
+ -X PUT \
+ --connect-timeout 1 \
+ -H "X-aws-ec2-metadata-token-ttl-seconds: 600" \
+ http://169.254.169.254/latest/api/token
+}
+
+preflight_imds_token() {
+ # retry-delay of 1 selected to give the system a second to get going,
+ # but not add a lot to the bootup time
+ curl \
+ --silent \
+ --show-error \
+ --retry 3 \
+ --retry-delay 1 \
+ --fail \
+ --connect-timeout 1 \
+ -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" \
+ -o /dev/null \
+ http://169.254.169.254/1.0/meta-data/instance-id
+}
+
+try=1
+while [ $try -le 3 ]; do
+ echo "(attempt $try/3) getting an EC2 instance metadata service v2 token..."
+ IMDS_TOKEN=$(get_imds_token) && break
+ try=$((try + 1))
+ sleep 1
+done
+
+if [ "x$IMDS_TOKEN" == "x" ]; then
+ echo "failed to fetch an IMDS2v token."
+fi
+
+try=1
+while [ $try -le 10 ]; do
+ echo "(attempt $try/10) validating the EC2 instance metadata service v2 token..."
+ preflight_imds_token && break
+ try=$((try + 1))
+ sleep 1
+done
+
+echo "getting EC2 instance metadata..."
+
+get_imds() {
+ # Intentionally no --fail here, so that we proceed even if e.g. a
+ # 404 was returned (but we still fail if we can't reach the IMDS
+ # server).
+ curl --silent --show-error --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@"
+}
+
+get_imds -o "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
+(umask 077 && get_imds -o "$metaDir/user-data" http://169.254.169.254/1.0/user-data)
+get_imds -o "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
+get_imds -o "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
diff --git a/nixos/tests/ec2.nix b/nixos/tests/ec2.nix
index aa3c2b7051f6..e649761d029d 100644
--- a/nixos/tests/ec2.nix
+++ b/nixos/tests/ec2.nix
@@ -16,8 +16,6 @@ let
../modules/testing/test-instrumentation.nix
../modules/profiles/qemu-guest.nix
{
- ec2.hvm = true;
-
# Hack to make the partition resizing work in QEMU.
boot.initrd.postDeviceCommands = mkBefore ''
ln -s vda /dev/xvda