gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/tests/nebula: test IPv6 connectivity (#468305)

Browse Source
This commit is contained in:
@mjones
2025-12-23 22:15:55 +00:00
committed by GitHub
parent e2ff99dbeb 42aa95303a
commit c6950ae6be
2 changed files with 159 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/tests/nebula/connectivity.nix
+158 -42
View File
@@ -16,6 +16,7 @@ let
environment.systemPackages = [
pkgs.dig
pkgs.nebula
pkgs.jq
];
users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
services.openssh.enable = true;
@@ -28,7 +29,7 @@ let
cert = "/etc/nebula/${name}.crt";
key = "/etc/nebula/${name}.key";
listen = {
host = "0.0.0.0";
host = "::";
port =
if
(
@@ -54,12 +55,20 @@ in
{ ... }@args:
makeNebulaNode args "lighthouse" {
networking.firewall.allowedUDPPorts = [ 53 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.1";
prefixLength = 24;
}
];
networking.interfaces.eth1 = {
ipv4.addresses = lib.mkForce [
{
address = "192.168.1.1";
prefixLength = 24;
}
];
ipv6.addresses = lib.mkForce [
{
address = "3fff::1";
prefixLength = 64;
}
];
};
services.nebula.networks.smoke = {
isLighthouse = true;
@@ -93,16 +102,27 @@ in
allowAny =
{ ... }@args:
makeNebulaNode args "allowAny" {
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.2";
prefixLength = 24;
}
];
networking.interfaces.eth1 = {
ipv4.addresses = lib.mkForce [
{
address = "192.168.1.2";
prefixLength = 24;
}
];
ipv6.addresses = lib.mkForce [
{
address = "3fff::2";
prefixLength = 64;
}
];
};
services.nebula.networks.smoke = {
staticHostMap = {
"10.0.100.1" = [ "192.168.1.1:4242" ];
"10.0.100.1" = [
"192.168.1.1:4242"
"[3fff::1]:4242"
];
};
isLighthouse = false;
lighthouses = [ "10.0.100.1" ];
@@ -129,16 +149,21 @@ in
allowFromLighthouse =
{ ... }@args:
makeNebulaNode args "allowFromLighthouse" {
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.3";
prefixLength = 24;
}
];
networking.interfaces.eth1 = {
ipv6.addresses = lib.mkForce [
{
address = "3fff::3";
prefixLength = 64;
}
];
};
services.nebula.networks.smoke = {
staticHostMap = {
"10.0.100.1" = [ "192.168.1.1:4242" ];
"10.0.100.1" = [
"192.168.1.1:4242"
"[3fff::1]:4242"
];
};
isLighthouse = false;
lighthouses = [ "10.0.100.1" ];
@@ -175,7 +200,10 @@ in
services.nebula.networks.smoke = {
enable = true;
staticHostMap = {
"10.0.100.1" = [ "192.168.1.1:4242" ];
"10.0.100.1" = [
"192.168.1.1:4242"
"[3fff::1]:4242"
];
};
isLighthouse = false;
lighthouses = [ "10.0.100.1" ];
@@ -212,7 +240,9 @@ in
services.nebula.networks.smoke = {
enable = false;
staticHostMap = {
"10.0.100.1" = [ "192.168.1.1:4242" ];
"10.0.100.1" = [
"192.168.1.1:4242"
];
};
isLighthouse = false;
lighthouses = [ "10.0.100.1" ];
@@ -270,7 +300,7 @@ in
"scp ${sshOpts} /etc/nebula/${name}.pub root@192.168.1.1:/root/${name}.pub",
)
lighthouse.succeed(
'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt'
'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -networks "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt'
)
${name}.succeed(
"scp ${sshOpts} root@192.168.1.1:/root/${name}.crt /etc/nebula/${name}.crt",
@@ -279,28 +309,57 @@ in
)
'';
getPublicIp = node: ''
${node}.succeed("ip --brief addr show eth1 | awk '{print $3}' | tail -n1 | cut -d/ -f1").strip()
getPublicIpv4 = node: ''
${node}.succeed("ip --json addr show eth1 | jq -r '.[0].addr_info | map(select(.family == \"inet\")) | map(.local) | join(\",\")'").strip()
'';
getPublicIpv6 = node: ''
${node}.succeed("ip --json addr show eth1 | jq -r '.[0].addr_info | map(select(.family == \"inet6\")) | map(.local) | join(\",\")'").strip()
'';
# Never do this for anything security critical! (Thankfully it's just a test.)
# Restart Nebula right after the mutual block and/or restore so the state is fresh.
blockTrafficBetween = nodeA: nodeB: ''
node_a = ${getPublicIp nodeA}
node_b = ${getPublicIp nodeB}
${nodeA}.succeed("iptables -I INPUT -s " + node_b + " -j DROP")
${nodeB}.succeed("iptables -I INPUT -s " + node_a + " -j DROP")
blockTrafficBetweenV4 = nodeA: nodeB: ''
node_a_4 = ${getPublicIpv4 nodeA}
node_b_4 = ${getPublicIpv4 nodeB}
${nodeA}.succeed("iptables -I INPUT -s " + node_b_4 + " -j DROP")
${nodeB}.succeed("iptables -I INPUT -s " + node_a_4 + " -j DROP")
${nodeA}.systemctl("restart nebula@smoke.service")
${nodeB}.systemctl("restart nebula@smoke.service")
'';
allowTrafficBetween = nodeA: nodeB: ''
node_a = ${getPublicIp nodeA}
node_b = ${getPublicIp nodeB}
${nodeA}.succeed("iptables -D INPUT -s " + node_b + " -j DROP")
${nodeB}.succeed("iptables -D INPUT -s " + node_a + " -j DROP")
allowTrafficBetweenV4 = nodeA: nodeB: ''
node_a_4 = ${getPublicIpv4 nodeA}
node_b_4 = ${getPublicIpv4 nodeB}
${nodeA}.succeed("iptables -D INPUT -s " + node_b_4 + " -j DROP")
${nodeB}.succeed("iptables -D INPUT -s " + node_a_4 + " -j DROP")
${nodeA}.systemctl("restart nebula@smoke.service")
${nodeB}.systemctl("restart nebula@smoke.service")
'';
blockTrafficBetweenV6 = nodeA: nodeB: ''
node_a_6 = ${getPublicIpv6 nodeA}
node_b_6 = ${getPublicIpv6 nodeB}
${nodeA}.succeed("ip6tables -I INPUT -i eth1 -s " + node_b_6 + " -j DROP")
${nodeB}.succeed("ip6tables -I INPUT -i eth1 -s " + node_a_6 + " -j DROP")
${nodeA}.systemctl("restart nebula@smoke.service")
${nodeB}.systemctl("restart nebula@smoke.service")
'';
allowTrafficBetweenV6 = nodeA: nodeB: ''
node_a_6 = ${getPublicIpv6 nodeA}
node_b_6 = ${getPublicIpv6 nodeB}
${nodeA}.succeed("ip6tables -D INPUT -i eth1 -s " + node_b_6 + " -j DROP")
${nodeB}.succeed("ip6tables -D INPUT -i eth1 -s " + node_a_6 + " -j DROP")
${nodeA}.systemctl("restart nebula@smoke.service")
${nodeB}.systemctl("restart nebula@smoke.service")
'';
blockTrafficBetween = nodeA: nodeB: ''
${blockTrafficBetweenV4 nodeA nodeB}
${blockTrafficBetweenV6 nodeA nodeB}
'';
allowTrafficBetween = nodeA: nodeB: ''
${allowTrafficBetweenV4 nodeA nodeB}
${allowTrafficBetweenV6 nodeA nodeB}
'';
in
''
# Create the certificate and sign the lighthouse's keys.
@@ -308,7 +367,7 @@ in
lighthouse.succeed(
"mkdir -p /etc/nebula",
'nebula-cert ca -duration $((10*365*24*60))m -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -networks "10.0.100.1/24,2001:db8::1/64" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
'chown -R nebula-smoke:nebula-smoke /etc/nebula'
)
@@ -318,6 +377,7 @@ in
lighthouse.start()
lighthouse.wait_for_unit("nebula@smoke.service")
lighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10)
lighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10)
# Start all the machines to be set up
allowAny.start()
@@ -327,101 +387,157 @@ in
# Create keys for allowAny's nebula service and test that it comes up.
${setUpPrivateKey "allowAny"}
${signKeysFor "allowAny" "10.0.100.2/24"}
${signKeysFor "allowAny" "10.0.100.2/24,2001:db8::2/64"}
${restartAndCheckNebula "allowAny" "10.0.100.2"}
${restartAndCheckNebula "allowAny" "2001:db8::2"}
# Create keys for allowFromLighthouse's nebula service and test that it comes up.
${setUpPrivateKey "allowFromLighthouse"}
${signKeysFor "allowFromLighthouse" "10.0.100.3/24"}
${signKeysFor "allowFromLighthouse" "10.0.100.3/24,2001:db8::3/64"}
${restartAndCheckNebula "allowFromLighthouse" "10.0.100.3"}
${restartAndCheckNebula "allowFromLighthouse" "2001:db8::3"}
# Create keys for allowToLighthouse's nebula service and test that it comes up.
${setUpPrivateKey "allowToLighthouse"}
${signKeysFor "allowToLighthouse" "10.0.100.4/24"}
${signKeysFor "allowToLighthouse" "10.0.100.4/24,2001:db8::4/64"}
${restartAndCheckNebula "allowToLighthouse" "10.0.100.4"}
${restartAndCheckNebula "allowToLighthouse" "2001:db8::4"}
# Create keys for disabled's nebula service and test that it does not come up.
${setUpPrivateKey "disabled"}
${signKeysFor "disabled" "10.0.100.5/24"}
${signKeysFor "disabled" "10.0.100.5/24,2001:db8::5/64"}
disabled.fail("systemctl status nebula@smoke.service")
disabled.fail("ping -c3 -W1 10.0.100.5")
disabled.fail("ping -c3 -W1 2001:db8::5")
# The lighthouse can ping allowAny and allowFromLighthouse but not disabled
lighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
lighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
lighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10)
lighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10)
lighthouse.fail("ping -c3 -W1 10.0.100.5")
lighthouse.fail("ping -c3 -W1 2001:db8::5")
# allowAny can ping the lighthouse, but not allowFromLighthouse because of its inbound firewall
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10)
allowAny.fail("ping -c3 -W1 10.0.100.3")
allowAny.fail("ping -c3 -W1 2001:db8::3")
# allowAny can also resolve DNS on lighthouse
allowAny.succeed("dig @10.0.100.1 allowToLighthouse | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+A\s+10\.0\.100\.4'")
allowAny.succeed("dig @10.0.100.1 allowToLighthouse A | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+A\s+10\.0\.100\.4'")
allowAny.succeed("dig @10.0.100.1 allowToLighthouse AAAA | grep -E 'allowToLighthouse\.\s+[0-9]+\s+IN\s+AAAA\s+2001:db8::4'")
# allowFromLighthouse can ping the lighthouse and allowAny
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
# allowAny does IPv6 -> IPv4 and IPv4 -> IPv6 switchover for the underlay network
${blockTrafficBetweenV4 "lighthouse" "allowAny"}
${blockTrafficBetweenV6 "lighthouse" "allowToLighthouse"}
${blockTrafficBetweenV6 "allowAny" "allowToLighthouse"}
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10)
${blockTrafficBetweenV6 "lighthouse" "allowAny"}
allowAny.fail("ping -c3 -W1 10.0.100.1")
allowAny.fail("ping -c3 -W1 2001:db8::4")
${allowTrafficBetweenV4 "lighthouse" "allowAny"}
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10)
${allowTrafficBetweenV6 "lighthouse" "allowAny"}
${allowTrafficBetweenV6 "lighthouse" "allowToLighthouse"}
${allowTrafficBetweenV6 "allowAny" "allowToLighthouse"}
# block allowFromLighthouse <-> allowAny, and allowFromLighthouse -> allowAny should still work.
${blockTrafficBetween "allowFromLighthouse" "allowAny"}
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
${allowTrafficBetween "allowFromLighthouse" "allowAny"}
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
# allowToLighthouse can ping the lighthouse but not allowAny or allowFromLighthouse
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.1", timeout=10)
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::1", timeout=10)
allowToLighthouse.fail("ping -c3 -W1 10.0.100.2")
allowToLighthouse.fail("ping -c3 -W1 2001:db8::2")
allowToLighthouse.fail("ping -c3 -W1 10.0.100.3")
allowToLighthouse.fail("ping -c3 -W1 2001:db8::3")
# allowAny can ping allowFromLighthouse now that allowFromLighthouse pinged it first
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10)
# block allowAny <-> allowFromLighthouse, and allowAny -> allowFromLighthouse should still work.
${blockTrafficBetween "allowAny" "allowFromLighthouse"}
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10)
${allowTrafficBetween "allowAny" "allowFromLighthouse"}
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10)
# allowToLighthouse can ping allowAny if allowAny pings it first
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10)
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
# block allowToLighthouse <-> allowAny, and allowAny <-> allowToLighthouse should still work.
${blockTrafficBetween "allowAny" "allowToLighthouse"}
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10)
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
${allowTrafficBetween "allowAny" "allowToLighthouse"}
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10)
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowToLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
# block lighthouse <-> allowFromLighthouse and allowAny <-> allowFromLighthouse; allowFromLighthouse won't get to allowAny
${blockTrafficBetween "allowFromLighthouse" "lighthouse"}
${blockTrafficBetween "allowFromLighthouse" "allowAny"}
allowFromLighthouse.fail("ping -c3 -W1 10.0.100.2")
allowFromLighthouse.fail("ping -c3 -W1 2001:db8::2")
${allowTrafficBetween "allowFromLighthouse" "lighthouse"}
${allowTrafficBetween "allowFromLighthouse" "allowAny"}
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
# block lighthouse <-> allowAny, allowAny <-> allowFromLighthouse, and allowAny <-> allowToLighthouse; it won't get to allowFromLighthouse or allowToLighthouse
${blockTrafficBetween "allowAny" "lighthouse"}
${blockTrafficBetween "allowAny" "allowFromLighthouse"}
${blockTrafficBetween "allowAny" "allowToLighthouse"}
allowFromLighthouse.fail("ping -c3 -W1 10.0.100.2")
allowFromLighthouse.fail("ping -c3 -W1 2001:db8::2")
allowAny.fail("ping -c3 -W1 10.0.100.3")
allowAny.fail("ping -c3 -W1 2001:db8::3")
allowAny.fail("ping -c3 -W1 10.0.100.4")
allowAny.fail("ping -c3 -W1 2001:db8::4")
${allowTrafficBetween "allowAny" "lighthouse"}
${allowTrafficBetween "allowAny" "allowFromLighthouse"}
${allowTrafficBetween "allowAny" "allowToLighthouse"}
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 10.0.100.2", timeout=10)
allowFromLighthouse.wait_until_succeeds("ping -c1 -W1 2001:db8::2", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.3", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::3", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10)
allowAny.wait_until_succeeds("ping -c1 -W1 2001:db8::4", timeout=10)
# block lighthouse <-> allowToLighthouse and allowToLighthouse <-> allowAny; it won't get to allowAny
${blockTrafficBetween "allowToLighthouse" "lighthouse"}
${blockTrafficBetween "allowToLighthouse" "allowAny"}
allowAny.fail("ping -c3 -W1 10.0.100.4")
allowAny.fail("ping -c3 -W1 2001:db8::4")
allowToLighthouse.fail("ping -c3 -W1 10.0.100.2")
allowToLighthouse.fail("ping -c3 -W1 2001:db8::2")
${allowTrafficBetween "allowToLighthouse" "lighthouse"}
${allowTrafficBetween "allowToLighthouse" "allowAny"}
allowAny.wait_until_succeeds("ping -c1 -W1 10.0.100.4", timeout=10)
nixos/tests/nebula/reload.nix
+1 -1
View File
@@ -66,7 +66,7 @@ in
lighthouse.succeed(
"mkdir -p /etc/nebula",
'nebula-cert ca -duration $((10*365*24*60))m -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
'nebula-cert sign -duration $((365*24*60))m -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -networks "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
'chown -R nebula-smoke:nebula-smoke /etc/nebula'
)