diff --git a/nixos/tests/nebula/connectivity.nix b/nixos/tests/nebula/connectivity.nix index 6bb6547dc89d..e58c4668a241 100644 --- a/nixos/tests/nebula/connectivity.nix +++ b/nixos/tests/nebula/connectivity.nix @@ -16,6 +16,7 @@ let environment.systemPackages = [ pkgs.dig pkgs.nebula + pkgs.jq ]; users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ]; services.openssh.enable = true; @@ -28,7 +29,7 @@ let cert = "/etc/nebula/${name}.crt"; key = "/etc/nebula/${name}.key"; listen = { - host = "0.0.0.0"; + host = "::"; port = if ( @@ -54,12 +55,20 @@ in { ... }@args: makeNebulaNode args "lighthouse" { networking.firewall.allowedUDPPorts = [ 53 ]; - networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ - { - address = "192.168.1.1"; - prefixLength = 24; - } - ]; + networking.interfaces.eth1 = { + ipv4.addresses = lib.mkForce [ + { + address = "192.168.1.1"; + prefixLength = 24; + } + ]; + ipv6.addresses = lib.mkForce [ + { + address = "3fff::1"; + prefixLength = 64; + } + ]; + }; services.nebula.networks.smoke = { isLighthouse = true; @@ -93,16 +102,27 @@ in allowAny = { ... }@args: makeNebulaNode args "allowAny" { - networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ - { - address = "192.168.1.2"; - prefixLength = 24; - } - ]; + networking.interfaces.eth1 = { + ipv4.addresses = lib.mkForce [ + { + address = "192.168.1.2"; + prefixLength = 24; + } + ]; + ipv6.addresses = lib.mkForce [ + { + address = "3fff::2"; + prefixLength = 64; + } + ]; + }; services.nebula.networks.smoke = { staticHostMap = { - "10.0.100.1" = [ "192.168.1.1:4242" ]; + "10.0.100.1" = [ + "192.168.1.1:4242" + "[3fff::1]:4242" + ]; }; isLighthouse = false; lighthouses = [ "10.0.100.1" ]; @@ -129,16 +149,21 @@ in allowFromLighthouse = { ... }@args: makeNebulaNode args "allowFromLighthouse" { - networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ - { - address = "192.168.1.3"; - prefixLength = 24; - } - ]; + networking.interfaces.eth1 = { + ipv6.addresses = lib.mkForce [ + { + address = "3fff::3"; + prefixLength = 64; + } + ]; + }; services.nebula.networks.smoke = { staticHostMap = { - "10.0.100.1" = [ "192.168.1.1:4242" ]; + "10.0.100.1" = [ + "192.168.1.1:4242" + "[3fff::1]:4242" + ]; }; isLighthouse = false; lighthouses = [ "10.0.100.1" ]; @@ -175,7 +200,10 @@ in services.nebula.networks.smoke = { enable = true; staticHostMap = { - "10.0.100.1" = [ "192.168.1.1:4242" ]; + "10.0.100.1" = [ + "192.168.1.1:4242" + "[3fff::1]:4242" + ]; }; isLighthouse = false; lighthouses = [ "10.0.100.1" ]; @@ -212,7 +240,9 @@ in services.nebula.networks.smoke = { enable = false; staticHostMap = { - "10.0.100.1" = [ "192.168.1.1:4242" ]; + "10.0.100.1" = [ + "192.168.1.1:4242" + ]; }; isLighthouse = false; lighthouses = [ "10.0.100.1" ]; @@ -270,7 +300,7 @@ in "scp ${sshOpts} /etc/nebula/${name}.pub root@192.168.1.1:/root/${name}.pub", ) lighthouse.succeed( - 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt' + 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -networks "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt' ) ${name}.succeed( "scp ${sshOpts} root@192.168.1.1:/root/${name}.crt /etc/nebula/${name}.crt", @@ -279,28 +309,57 @@ in ) ''; - getPublicIp = node: '' - ${node}.succeed("ip --brief addr show eth1 | awk '{print $3}' | tail -n1 | cut -d/ -f1").strip() + getPublicIpv4 = node: '' + ${node}.succeed("ip --json addr show eth1 | jq -r '.[0].addr_info | map(select(.family == \"inet\")) | map(.local) | join(\",\")'").strip() + ''; + + getPublicIpv6 = node: '' + ${node}.succeed("ip --json addr show eth1 | jq -r '.[0].addr_info | map(select(.family == \"inet6\")) | map(.local) | join(\",\")'").strip() ''; # Never do this for anything security critical! (Thankfully it's just a test.) # Restart Nebula right after the mutual block and/or restore so the state is fresh. - blockTrafficBetween = nodeA: nodeB: '' - node_a = ${getPublicIp nodeA} - node_b = ${getPublicIp nodeB} - ${nodeA}.succeed("iptables -I INPUT -s " + node_b + " -j DROP") - ${nodeB}.succeed("iptables -I INPUT -s " + node_a + " -j DROP") + blockTrafficBetweenV4 = nodeA: nodeB: '' + node_a_4 = ${getPublicIpv4 nodeA} + node_b_4 = ${getPublicIpv4 nodeB} + ${nodeA}.succeed("iptables -I INPUT -s " + node_b_4 + " -j DROP") + ${nodeB}.succeed("iptables -I INPUT -s " + node_a_4 + " -j DROP") ${nodeA}.systemctl("restart nebula@smoke.service") ${nodeB}.systemctl("restart nebula@smoke.service") ''; - allowTrafficBetween = nodeA: nodeB: '' - node_a = ${getPublicIp nodeA} - node_b = ${getPublicIp nodeB} - ${nodeA}.succeed("iptables -D INPUT -s " + node_b + " -j DROP") - ${nodeB}.succeed("iptables -D INPUT -s " + node_a + " -j DROP") + allowTrafficBetweenV4 = nodeA: nodeB: '' + node_a_4 = ${getPublicIpv4 nodeA} + node_b_4 = ${getPublicIpv4 nodeB} + ${nodeA}.succeed("iptables -D INPUT -s " + node_b_4 + " -j DROP") + ${nodeB}.succeed("iptables -D INPUT -s " + node_a_4 + " -j DROP") ${nodeA}.systemctl("restart nebula@smoke.service") ${nodeB}.systemctl("restart nebula@smoke.service") ''; + blockTrafficBetweenV6 = nodeA: nodeB: '' + node_a_6 = ${getPublicIpv6 nodeA} + node_b_6 = ${getPublicIpv6 nodeB} + ${nodeA}.succeed("ip6tables -I INPUT -i eth1 -s " + node_b_6 + " -j DROP") + ${nodeB}.succeed("ip6tables -I INPUT -i eth1 -s " + node_a_6 + " -j DROP") + ${nodeA}.systemctl("restart nebula@smoke.service") + ${nodeB}.systemctl("restart nebula@smoke.service") + ''; + allowTrafficBetweenV6 = nodeA: nodeB: '' + node_a_6 = ${getPublicIpv6 nodeA} + node_b_6 = ${getPublicIpv6 nodeB} + ${nodeA}.succeed("ip6tables -D INPUT -i eth1 -s " + node_b_6 + " -j DROP") + ${nodeB}.succeed("ip6tables -D INPUT -i eth1 -s " + node_a_6 + " -j DROP") + ${nodeA}.systemctl("restart nebula@smoke.service") + ${nodeB}.systemctl("restart nebula@smoke.service") + ''; + + blockTrafficBetween = nodeA: nodeB: '' + ${blockTrafficBetweenV4 nodeA nodeB} + ${blockTrafficBetweenV6 nodeA nodeB} + ''; + allowTrafficBetween = nodeA: nodeB: '' + ${allowTrafficBetweenV4 nodeA nodeB} + ${allowTrafficBetweenV6 nodeA nodeB} + ''; in '' # Create the certificate and sign the lighthouse's keys. @@ -308,7 +367,7 @@ in lighthouse.succeed( "mkdir -p /etc/nebula", 'nebula-cert ca -duration $((10*365*24*60))m -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key', - 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key', + 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -networks "10.0.100.1/24,2001:db8::1/64" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key', 'chown -R nebula-smoke:nebula-smoke /etc/nebula' ) @@ -318,6 +377,7 @@ in lighthouse.start() lighthouse.wait_for_unit("nebula@smoke.service") lighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10) + lighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10) # Start all the machines to be set up allowAny.start() @@ -327,101 +387,157 @@ in # Create keys for allowAny's nebula service and test that it comes up. ${setUpPrivateKey "allowAny"} - ${signKeysFor "allowAny" "10.0.100.2/24"} + ${signKeysFor "allowAny" "10.0.100.2/24,2001:db8::2/64"} ${restartAndCheckNebula "allowAny" "10.0.100.2"} + ${restartAndCheckNebula "allowAny" "2001:db8::2"} # Create keys for allowFromLighthouse's nebula service and test that it comes up. ${setUpPrivateKey "allowFromLighthouse"} - ${signKeysFor "allowFromLighthouse" "10.0.100.3/24"} + ${signKeysFor "allowFromLighthouse" "10.0.100.3/24,2001:db8::3/64"} ${restartAndCheckNebula "allowFromLighthouse" "10.0.100.3"} + ${restartAndCheckNebula "allowFromLighthouse" "2001:db8::3"} # Create keys for allowToLighthouse's nebula service and test that it comes up. ${setUpPrivateKey "allowToLighthouse"} - ${signKeysFor "allowToLighthouse" "10.0.100.4/24"} + ${signKeysFor "allowToLighthouse" "10.0.100.4/24,2001:db8::4/64"} ${restartAndCheckNebula "allowToLighthouse" "10.0.100.4"} + ${restartAndCheckNebula "allowToLighthouse" "2001:db8::4"} # Create keys for disabled's nebula service and test that it does not come up. ${setUpPrivateKey "disabled"} - ${signKeysFor "disabled" "10.0.100.5/24"} + ${signKeysFor "disabled" "10.0.100.5/24,2001:db8::5/64"} disabled.fail("systemctl status nebula@smoke.service") disabled.fail("ping -c3 -W1 10.0.100.5") + disabled.fail("ping -c3 -W1 2001:db8::5") # The lighthouse can ping allowAny and allowFromLighthouse but not disabled lighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + lighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) lighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10) + lighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10) lighthouse.fail("ping -c3 -W1 10.0.100.5") + lighthouse.fail("ping -c3 -W1 2001:db8::5") # allowAny can ping the lighthouse, but not allowFromLighthouse because of its inbound firewall allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10) allowAny.fail("ping -c3 -W1 10.0.100.3") + allowAny.fail("ping -c3 -W1 2001:db8::3") # allowAny can also resolve DNS on lighthouse - allowAny.succeed("dig @10.0.100.1 allowToLighthouse | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+A\s+10\.0\.100\.4'") + allowAny.succeed("dig @10.0.100.1 allowToLighthouse A | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+A\s+10\.0\.100\.4'") + allowAny.succeed("dig @10.0.100.1 allowToLighthouse AAAA | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+AAAA\s+2001:db8::4'") # allowFromLighthouse can ping the lighthouse and allowAny allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10) + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10) allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) + + # allowAny does IPv6 -> IPv4 and IPv4 -> IPv6 switchover for the underlay network + ${blockTrafficBetweenV4 "lighthouse" "allowAny"} + ${blockTrafficBetweenV6 "lighthouse" "allowToLighthouse"} + ${blockTrafficBetweenV6 "allowAny" "allowToLighthouse"} + allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10) + ${blockTrafficBetweenV6 "lighthouse" "allowAny"} + allowAny.fail("ping -c3 -W1 10.0.100.1") + allowAny.fail("ping -c3 -W1 2001:db8::4") + ${allowTrafficBetweenV4 "lighthouse" "allowAny"} + allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10) + ${allowTrafficBetweenV6 "lighthouse" "allowAny"} + ${allowTrafficBetweenV6 "lighthouse" "allowToLighthouse"} + ${allowTrafficBetweenV6 "allowAny" "allowToLighthouse"} # block allowFromLighthouse <-> allowAny, and allowFromLighthouse -> allowAny should still work. ${blockTrafficBetween "allowFromLighthouse" "allowAny"} + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) ${allowTrafficBetween "allowFromLighthouse" "allowAny"} allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) # allowToLighthouse can ping the lighthouse but not allowAny or allowFromLighthouse allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10) + allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10) allowToLighthouse.fail("ping -c3 -W1 10.0.100.2") + allowToLighthouse.fail("ping -c3 -W1 2001:db8::2") allowToLighthouse.fail("ping -c3 -W1 10.0.100.3") + allowToLighthouse.fail("ping -c3 -W1 2001:db8::3") # allowAny can ping allowFromLighthouse now that allowFromLighthouse pinged it first allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10) # block allowAny <-> allowFromLighthouse, and allowAny -> allowFromLighthouse should still work. ${blockTrafficBetween "allowAny" "allowFromLighthouse"} allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10) ${allowTrafficBetween "allowAny" "allowFromLighthouse"} allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10) # allowToLighthouse can ping allowAny if allowAny pings it first allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10) allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) # block allowToLighthouse <-> allowAny, and allowAny <-> allowToLighthouse should still work. ${blockTrafficBetween "allowAny" "allowToLighthouse"} allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10) allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) ${allowTrafficBetween "allowAny" "allowToLighthouse"} allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10) allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) # block lighthouse <-> allowFromLighthouse and allowAny <-> allowFromLighthouse; allowFromLighthouse won't get to allowAny ${blockTrafficBetween "allowFromLighthouse" "lighthouse"} ${blockTrafficBetween "allowFromLighthouse" "allowAny"} allowFromLighthouse.fail("ping -c3 -W1 10.0.100.2") + allowFromLighthouse.fail("ping -c3 -W1 2001:db8::2") ${allowTrafficBetween "allowFromLighthouse" "lighthouse"} ${allowTrafficBetween "allowFromLighthouse" "allowAny"} allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) # block lighthouse <-> allowAny, allowAny <-> allowFromLighthouse, and allowAny <-> allowToLighthouse; it won't get to allowFromLighthouse or allowToLighthouse ${blockTrafficBetween "allowAny" "lighthouse"} ${blockTrafficBetween "allowAny" "allowFromLighthouse"} ${blockTrafficBetween "allowAny" "allowToLighthouse"} allowFromLighthouse.fail("ping -c3 -W1 10.0.100.2") + allowFromLighthouse.fail("ping -c3 -W1 2001:db8::2") allowAny.fail("ping -c3 -W1 10.0.100.3") + allowAny.fail("ping -c3 -W1 2001:db8::3") allowAny.fail("ping -c3 -W1 10.0.100.4") + allowAny.fail("ping -c3 -W1 2001:db8::4") ${allowTrafficBetween "allowAny" "lighthouse"} ${allowTrafficBetween "allowAny" "allowFromLighthouse"} ${allowTrafficBetween "allowAny" "allowToLighthouse"} allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10) + allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10) allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10) allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10) + allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10) # block lighthouse <-> allowToLighthouse and allowToLighthouse <-> allowAny; it won't get to allowAny ${blockTrafficBetween "allowToLighthouse" "lighthouse"} ${blockTrafficBetween "allowToLighthouse" "allowAny"} allowAny.fail("ping -c3 -W1 10.0.100.4") + allowAny.fail("ping -c3 -W1 2001:db8::4") allowToLighthouse.fail("ping -c3 -W1 10.0.100.2") + allowToLighthouse.fail("ping -c3 -W1 2001:db8::2") ${allowTrafficBetween "allowToLighthouse" "lighthouse"} ${allowTrafficBetween "allowToLighthouse" "allowAny"} allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10) diff --git a/nixos/tests/nebula/reload.nix b/nixos/tests/nebula/reload.nix index 5e42cb2150a4..1495dbadc473 100644 --- a/nixos/tests/nebula/reload.nix +++ b/nixos/tests/nebula/reload.nix @@ -66,7 +66,7 @@ in lighthouse.succeed( "mkdir -p /etc/nebula", 'nebula-cert ca -duration $((10*365*24*60))m -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key', - 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key', + 'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -networks "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key', 'chown -R nebula-smoke:nebula-smoke /etc/nebula' )