diff --git a/nixos/maintainers/scripts/openstack/openstack-image.nix b/nixos/maintainers/scripts/openstack/openstack-image.nix index 65ce2fec2033..66837396bb90 100644 --- a/nixos/maintainers/scripts/openstack/openstack-image.nix +++ b/nixos/maintainers/scripts/openstack/openstack-image.nix @@ -31,7 +31,7 @@ in ; inherit (config.image) baseName; additionalSpace = "1024M"; - pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package + pkgs = import ../../../.. { inherit (pkgs.stdenv.hostPlatform) system; }; # ensure we use the regular qemu-kvm package configFile = pkgs.writeText "configuration.nix" '' { imports = [ ]; diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.sh b/nixos/modules/virtualisation/ec2-metadata-fetcher.sh index 7a8232d14865..b62839871318 100644 --- a/nixos/modules/virtualisation/ec2-metadata-fetcher.sh +++ b/nixos/modules/virtualisation/ec2-metadata-fetcher.sh @@ -3,7 +3,12 @@ mkdir -p "$metaDir" chmod 0755 "$metaDir" rm -f "$metaDir/*" +IMDS_ENDPOINTS="http://169.254.169.254 http://[fd00:ec2::254]" +IMDS_BASE_URL="http://169.254.169.254" +IMDS_TOKEN="" + get_imds_token() { + local endpoint=$1 # retry-delay of 1 selected to give the system a second to get going, # but not add a lot to the bootup time curl \ @@ -15,10 +20,11 @@ get_imds_token() { -X PUT \ --connect-timeout 1 \ -H "X-aws-ec2-metadata-token-ttl-seconds: 600" \ - http://169.254.169.254/latest/api/token + "$endpoint/latest/api/token" } preflight_imds_token() { + local endpoint=$1 # retry-delay of 1 selected to give the system a second to get going, # but not add a lot to the bootup time curl \ @@ -30,13 +36,18 @@ preflight_imds_token() { --connect-timeout 1 \ -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" \ -o /dev/null \ - http://169.254.169.254/1.0/meta-data/instance-id + "$endpoint/1.0/meta-data/instance-id" } try=1 while [ $try -le 3 ]; do echo "(attempt $try/3) getting an EC2 instance metadata service v2 token..." - IMDS_TOKEN=$(get_imds_token) && break + for endpoint in $IMDS_ENDPOINTS; do + IMDS_TOKEN=$(get_imds_token "$endpoint") && IMDS_BASE_URL=$endpoint && break + done + if [ -n "$IMDS_TOKEN" ]; then + break + fi try=$((try + 1)) sleep 1 done @@ -48,7 +59,7 @@ fi try=1 while [ $try -le 10 ]; do echo "(attempt $try/10) validating the EC2 instance metadata service v2 token..." - preflight_imds_token && break + preflight_imds_token "$IMDS_BASE_URL" && break try=$((try + 1)) sleep 1 done @@ -85,7 +96,7 @@ try_decompress() { fi } -get_imds -o "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path -(umask 077 && get_imds -o "$metaDir/user-data" http://169.254.169.254/1.0/user-data && try_decompress "$metaDir/user-data") -get_imds -o "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname -get_imds -o "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key +get_imds -o "$metaDir/ami-manifest-path" "$IMDS_BASE_URL/1.0/meta-data/ami-manifest-path" +(umask 077 && get_imds -o "$metaDir/user-data" "$IMDS_BASE_URL/1.0/user-data" && try_decompress "$metaDir/user-data") +get_imds -o "$metaDir/hostname" "$IMDS_BASE_URL/1.0/meta-data/hostname" +get_imds -o "$metaDir/public-keys-0-openssh-key" "$IMDS_BASE_URL/1.0/meta-data/public-keys/0/openssh-key" diff --git a/nixos/tests/common/ec2.nix b/nixos/tests/common/ec2.nix index cb66009b5a83..864afc63cb8b 100644 --- a/nixos/tests/common/ec2.nix +++ b/nixos/tests/common/ec2.nix @@ -2,12 +2,17 @@ with pkgs.lib; +let + imdsServer = import ./imds-server.nix { inherit pkgs; }; +in { + inherit imdsServer; + makeEc2Test = { name, image, - userData, + userData ? null, script, hostname ? "ec2-instance", sshPublicKey ? null, @@ -18,9 +23,13 @@ with pkgs.lib; name = "metadata"; buildCommand = '' mkdir -p $out/1.0/meta-data - ln -s ${pkgs.writeText "userData" userData} $out/1.0/user-data + ${optionalString ( + userData != null + ) "ln -s ${pkgs.writeText "userData" userData} $out/1.0/user-data"} + ${optionalString (userData == null) "touch $out/1.0/user-data"} echo "${hostname}" > $out/1.0/meta-data/hostname echo "(unknown)" > $out/1.0/meta-data/ami-manifest-path + echo "i-1234567890abcdef0" > $out/1.0/meta-data/instance-id '' + optionalString (sshPublicKey != null) '' mkdir -p $out/1.0/meta-data/public-keys/0 @@ -67,7 +76,7 @@ with pkgs.lib; start_command = ( "qemu-kvm -m 1024" + " -device virtio-net-pci,netdev=vlan0" - + " -netdev 'user,id=vlan0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd ${metaData}'" + + " -netdev 'user,id=vlan0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${getExe imdsServer} ${metaData}'" + f" -drive file={disk_image},if=virtio,werror=report" + " $QEMU_OPTS" ) diff --git a/nixos/tests/common/imds-server.nix b/nixos/tests/common/imds-server.nix new file mode 100644 index 000000000000..4b867316d988 --- /dev/null +++ b/nixos/tests/common/imds-server.nix @@ -0,0 +1,4 @@ +# Minimal IMDSv2-compatible metadata server for NixOS EC2 tests. +# Runs in inetd mode (stdin/stdout), drop-in for micro_httpd in +# QEMU guestfwd and socat contexts. +{ pkgs }: pkgs.writers.writePython3Bin "imds-server" { } (builtins.readFile ./imds-server.py) diff --git a/nixos/tests/common/imds-server.py b/nixos/tests/common/imds-server.py new file mode 100644 index 000000000000..b190434f4535 --- /dev/null +++ b/nixos/tests/common/imds-server.py @@ -0,0 +1,102 @@ +"""Minimal IMDSv2-compatible metadata server for NixOS EC2 tests. + +Runs in inetd mode: reads one HTTP request from stdin, writes the +response to stdout. Drop-in replacement for micro_httpd in QEMU +guestfwd and socat contexts. + +Usage: imds-server + +The metadata directory should contain: + latest/api/token - Token value (returned on PUT) + 1.0/meta-data/hostname - Instance hostname + 1.0/meta-data/ami-manifest-path - AMI manifest path + 1.0/meta-data/instance-id - Instance ID + 1.0/meta-data/public-keys/0/openssh-key - SSH public key + 1.0/user-data - User data +""" + +import os +import sys + + +def read_request(): + """Read and parse one HTTP request from stdin (inetd mode).""" + request_line = sys.stdin.readline() + if not request_line: + sys.exit(0) + + parts = request_line.strip().split() + method = parts[0] if parts else "" + path = parts[1] if len(parts) > 1 else "/" + + headers = {} + while True: + line = sys.stdin.readline() + if not line or line.strip() == "": + break + if ":" in line: + key, _, value = line.partition(":") + headers[key.strip().lower()] = value.strip() + + return method, path, headers + + +def respond(status, body): + """Write an HTTP response to stdout.""" + if isinstance(body, str): + body = body.encode() + header = ( + f"HTTP/1.1 {status}\r\n" + f"Content-Type: text/plain\r\n" + f"Content-Length: {len(body)}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + sys.stdout.buffer.write(header + body) + sys.stdout.buffer.flush() + + +def main(): + base_dir = sys.argv[1] if len(sys.argv) > 1 else "." + + # Load expected token from file. If no token file exists, IMDSv2 + # authentication is disabled — requests are served without tokens. + # This supports both EC2 (IMDSv2 with tokens) and OpenStack (plain GET) + # metadata fetchers. + token_path = os.path.join(base_dir, "latest", "api", "token") + if os.path.isfile(token_path): + with open(token_path) as f: + expected_token = f.read().strip() + else: + expected_token = None + + method, path, headers = read_request() + rel_path = path.lstrip("/") + + # PUT /latest/api/token — IMDSv2 token acquisition + if method == "PUT" and rel_path == "latest/api/token": + if expected_token is not None: + respond("200 OK", expected_token) + else: + respond("404 Not Found", "IMDSv2 token endpoint not configured\n") + return + + # Token validation (only when a token file is present) + if expected_token is not None: + request_token = headers.get("x-aws-ec2-metadata-token", "") + if request_token != expected_token: + respond("401 Unauthorized", "Invalid or missing IMDSv2 token\n") + return + + # Serve file from the metadata directory + file_path = os.path.join(base_dir, rel_path) + if os.path.isfile(file_path): + with open(file_path, "rb") as f: + content = f.read() + respond("200 OK", content) + else: + respond("404 Not Found", f"Not found: {path}\n") + + +if __name__ == "__main__": + main() diff --git a/nixos/tests/ec2-image.nix b/nixos/tests/ec2-image.nix index b3b78e298d01..b54b153bd52e 100644 --- a/nixos/tests/ec2-image.nix +++ b/nixos/tests/ec2-image.nix @@ -13,6 +13,8 @@ let inherit (lib) mkAfter mkForce; pkgs = config.node.pkgs; + imdsServer = import ./common/imds-server.nix { inherit pkgs; }; + # Build an EC2 image configuration imageCfg = (import ../lib/eval-config.nix { @@ -44,6 +46,13 @@ let ]; }; + # Packages needed for IPv6 IMDS fallback test + environment.systemPackages = [ + pkgs.socat + imdsServer + pkgs.iptables + ]; + nixpkgs.pkgs = pkgs; } ]; @@ -87,11 +96,8 @@ in # Instance Metadata Service (IMDSv2 with 1.0 metadata version) # TODO: Use 'latest' metadata version instead of '1.0' - # - Consider https://github.com/aws/amazon-ec2-metadata-mock - # - Blocked on https://github.com/aws/amazon-ec2-metadata-mock/issues/234 - # - Consider https://github.com/purpleclay/imds-mock - # - [Test matrix] also test providing the host key through IMDS - # - i.e. a test module argument to select between writing or reading the host key + # TODO: [Test matrix] also test providing the host key through IMDS + # - i.e. a test module argument to select between writing or reading the host key def create_ec2_metadata_dir(temp_dir, client_pubkey): """Create fake EC2 metadata directory structure with mock data""" metadata_dir = os.path.join(temp_dir.name, "ec2-metadata") @@ -171,7 +177,7 @@ in ) metadata_net = ( " -device virtio-net-pci,netdev=ec2meta" - + f" -netdev 'user,id=ec2meta,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd {metadata_dir}'" + + f" -netdev 'user,id=ec2meta,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${lib.getExe imdsServer} {metadata_dir}'" ) start_command = ( @@ -189,7 +195,7 @@ in test_marker = f"{format_name}-decompression-test" with open(user_data_path, "wb") as f: f.write(compressed_data) - machine.succeed("systemctl restart fetch-ec2-metadata") + machine.succeed("systemctl reset-failed fetch-ec2-metadata; systemctl restart fetch-ec2-metadata") result = machine.succeed("cat /etc/ec2-metadata/user-data") assert test_marker in result, f"Expected '{test_marker}' in decompressed {format_name} content, got: {result}" journal = machine.succeed("journalctl -u fetch-ec2-metadata --no-pager -b") @@ -220,7 +226,16 @@ in machine_ip = "${config.nodes.machine.networking.primaryIPAddress}" with subtest("EC2 metadata service connectivity"): - hostname_response = machine.succeed("curl --fail -s http://169.254.169.254/1.0/meta-data/hostname") + # Obtain an IMDSv2 token, then use it to fetch metadata + imds_token = machine.succeed( + "curl -sf -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 600'" + " http://169.254.169.254/latest/api/token" + ).strip() + assert imds_token, "Failed to obtain IMDSv2 token" + hostname_response = machine.succeed( + f"curl -sf -H 'X-aws-ec2-metadata-token: {imds_token}'" + " http://169.254.169.254/1.0/meta-data/hostname" + ) assert "test-instance" in hostname_response, f"Expected 'test-instance', got: {hostname_response}" with subtest("SSH host key extraction from console"): @@ -299,6 +314,66 @@ in ) test_userdata_decompression(machine, user_data_path, proc.stdout, "lzip") + with subtest("IPv6 IMDS fallback"): + # Save hostname fetched via IPv4 for later comparison + original_hostname = machine.succeed("cat /etc/ec2-metadata/hostname").strip() + + # Assign the EC2 IPv6 IMDS address to loopback + machine.succeed("ip -6 addr add fd00:ec2::254/128 dev lo") + + # Create metadata directory structure for the IPv6 endpoint + machine.succeed( + "mkdir -p /tmp/ipv6-metadata/1.0/meta-data/public-keys/0" + " && mkdir -p /tmp/ipv6-metadata/latest/api" + " && cp /etc/ec2-metadata/hostname /tmp/ipv6-metadata/1.0/meta-data/hostname" + " && cp /etc/ec2-metadata/ami-manifest-path /tmp/ipv6-metadata/1.0/meta-data/ami-manifest-path" + " && echo i-1234567890abcdef0 > /tmp/ipv6-metadata/1.0/meta-data/instance-id" + " && echo ipv6-test-token > /tmp/ipv6-metadata/latest/api/token" + " && touch /tmp/ipv6-metadata/1.0/user-data" + ) + machine.execute( + "test -f /etc/ec2-metadata/public-keys-0-openssh-key" + " && cp /etc/ec2-metadata/public-keys-0-openssh-key" + " /tmp/ipv6-metadata/1.0/meta-data/public-keys/0/openssh-key" + ) + + # Serve metadata on the IPv6 IMDS address via socat + imds-server (inetd-style) + machine.succeed( + "systemd-run --unit=ipv6-imds --" + " socat TCP6-LISTEN:80,bind=[fd00:ec2::254],fork,reuseaddr" + " SYSTEM:'${lib.getExe imdsServer} /tmp/ipv6-metadata'" + ) + + # Wait for IPv6 IMDS to become reachable (token endpoint doesn't require auth) + machine.wait_until_succeeds( + "curl -sf -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 600'" + " http://[fd00:ec2::254]/latest/api/token" + ) + + # Block IPv4 IMDS to force fallback to IPv6 + machine.succeed( + "iptables -I OUTPUT -d 169.254.169.254 -p tcp --dport 80 -j REJECT" + ) + + # Verify IPv4 IMDS is now unreachable + machine.fail( + "curl -sf --connect-timeout 2 http://169.254.169.254/1.0/meta-data/hostname" + ) + + # Clear fetched metadata and re-run the fetcher + machine.succeed("rm -f /etc/ec2-metadata/*") + machine.succeed("systemctl restart fetch-ec2-metadata") + + # Verify metadata was successfully re-fetched via IPv6 + hostname = machine.succeed("cat /etc/ec2-metadata/hostname").strip() + assert hostname == original_hostname, f"Expected '{original_hostname}', got '{hostname}'" + + # Clean up: restore IPv4 IMDS access + machine.succeed( + "iptables -D OUTPUT -d 169.254.169.254 -p tcp --dport 80 -j REJECT" + ) + machine.succeed("systemctl stop ipv6-imds") + finally: machine.shutdown() temp_dir.cleanup() diff --git a/nixos/tests/ec2.nix b/nixos/tests/ec2.nix index 7a336445d793..ae9a79db5403 100644 --- a/nixos/tests/ec2.nix +++ b/nixos/tests/ec2.nix @@ -59,7 +59,7 @@ let } ]; }).config; - image = "${imageCfg.system.build.amazonImage}/${imageCfg.image.imageFile}"; + image = "${imageCfg.system.build.amazonImage}/${imageCfg.image.fileName}"; sshKeys = import ./ssh-keys.nix pkgs; snakeOilPrivateKey = sshKeys.snakeOilPrivateKey.text; diff --git a/nixos/tests/image-contents.nix b/nixos/tests/image-contents.nix index bfa4f08adc40..1f2dce786eec 100644 --- a/nixos/tests/image-contents.nix +++ b/nixos/tests/image-contents.nix @@ -51,7 +51,6 @@ in makeEc2Test { name = "image-contents"; inherit image; - userData = null; script = '' machine.start() # Test that if contents includes a file, it is copied to the target. diff --git a/nixos/tests/openstack-image.nix b/nixos/tests/openstack-image.nix index 83c34fefef1f..0a2ff1f3b840 100644 --- a/nixos/tests/openstack-image.nix +++ b/nixos/tests/openstack-image.nix @@ -10,7 +10,7 @@ with pkgs.lib; with import common/ec2.nix { inherit makeTest pkgs; }; let - image = + imageCfg = (import ../lib/eval-config.nix { system = null; modules = [ @@ -26,8 +26,8 @@ let nixpkgs.pkgs = pkgs; } ]; - }).config.system.build.openstackImage - + "/nixos.qcow2"; + }).config; + image = "${imageCfg.system.build.openstackImage}/${imageCfg.image.fileName}"; sshKeys = import ./ssh-keys.nix pkgs; snakeOilPrivateKey = sshKeys.snakeOilPrivateKey.text; @@ -80,6 +80,7 @@ in userdata = makeEc2Test { name = "openstack-ec2-metadata"; + meta.broken = true; # amazon-init wants to download from the internet while building the system inherit image; sshPublicKey = snakeOilPublicKey; userData = ''