gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

garage_2: init at 2.0.0, move tests to runTest (#417549)

Browse Source
This commit is contained in:
Adam C. Stephens
2025-06-22 16:15:39 -04:00
committed by GitHub
parent 0acbb41a19 b9700f766a
commit 0e6feffe10
7 changed files with 204 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/tests/all-tests.nix
+8 -1
View File
@@ -543,7 +543,14 @@ in
mimir = runTest ./mimir.nix;
galene = discoverTests (import ./galene.nix);
gancio = runTest ./gancio.nix;
garage = handleTest ./garage { };
garage_1 = import ./garage {
inherit runTest;
package = pkgs.garage_1;
};
garage_2 = import ./garage {
inherit runTest;
package = pkgs.garage_2;
};
gatus = runTest ./gatus.nix;
getaddrinfo = runTest ./getaddrinfo.nix;
gemstash = handleTest ./gemstash.nix { };
nixos/tests/garage/basic.nix
+28 -94
View File
@@ -1,106 +1,40 @@
args@{ mkNode, ver, ... }:
(import ../make-test-python.nix (
{ pkgs, ... }:
{
name = "garage-basic";
meta = {
maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
{
lib,
mkNode,
package,
testScriptSetup,
...
}:
{
name = "garage-basic";
nodes = {
single_node = mkNode {
extraSettings =
if (lib.versionAtLeast package.version "2") then
{
replication_factor = 1;
consistency_mode = "consistent";
}
else
{
replication_mode = "none";
};
};
};
nodes = {
single_node = mkNode { replicationMode = "none"; };
};
testScript = ''
from typing import List
from dataclasses import dataclass
import re
start_all()
cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')
key_creation_regex = re.compile('Key name: (?P<key_name>.*)\nKey ID: (?P<key_id>.*)\nSecret key: (?P<secret_key>.*)')
@dataclass
class S3Key:
key_name: str
key_id: str
secret_key: str
@dataclass
class GarageNode:
node_id: str
host: str
def get_node_fqn(machine: Machine) -> GarageNode:
node_id, host = machine.succeed("garage node id").split('@')
return GarageNode(node_id=node_id, host=host)
def get_node_id(machine: Machine) -> str:
return get_node_fqn(machine).node_id
def get_layout_version(machine: Machine) -> int:
version_data = machine.succeed("garage layout show")
m = cur_version_regex.search(version_data)
if m and m.group('ver') is not None:
return int(m.group('ver')) + 1
else:
raise ValueError('Cannot find current layout version')
def apply_garage_layout(machine: Machine, layouts: List[str]):
for layout in layouts:
machine.succeed(f"garage layout assign {layout}")
version = get_layout_version(machine)
machine.succeed(f"garage layout apply --version {version}")
def create_api_key(machine: Machine, key_name: str) -> S3Key:
output = machine.succeed(f"garage key ${
if ver == "0_8" then "new --name" else "create"
} {key_name}")
m = key_creation_regex.match(output)
if not m or not m.group('key_id') or not m.group('secret_key'):
raise ValueError('Cannot parse API key data')
return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))
def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
output = machine.succeed(f"garage key info {key_pattern}")
m = key_creation_regex.match(output)
if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):
raise ValueError('Cannot parse API key data')
return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))
def test_bucket_writes(node):
node.succeed("garage bucket create test-bucket")
s3_key = create_api_key(node, "test-api-key")
node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
other_s3_key = get_api_key(node, 'test-api-key')
assert other_s3_key.secret_key == other_s3_key.secret_key
node.succeed(
f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
)
node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")
assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"
def test_bucket_over_http(node, bucket='test-bucket', url=None):
if url is None:
url = f"{bucket}.web.garage"
node.succeed(f'garage bucket website --allow {bucket}')
node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')
assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
testScript = # python
''
${testScriptSetup}
with subtest("Garage works as a single-node S3 storage"):
single_node.wait_for_unit("garage.service")
single_node.wait_for_open_port(3900)
# Now Garage is initialized.
single_node_id = get_node_id(single_node)
apply_garage_layout(single_node, [f'-z qemutest -c ${
if ver == "0_8" then "1" else "1G"
} "{single_node_id}"'])
apply_garage_layout(single_node, [f'-z qemutest -c 1G "{single_node_id}"'])
# Now Garage is operational.
test_bucket_writes(single_node)
test_bucket_over_http(single_node)
'';
}
))
args
}
nixos/tests/garage/common.nix
+85
View File
@@ -0,0 +1,85 @@
{ ... }:
{
_module.args.testScriptSetup = # python
''
from typing import List
from dataclasses import dataclass
import re
start_all()
cur_version_regex = re.compile(r'Current cluster layout version: (?P<ver>\d*)')
@dataclass
class S3Key:
key_name: str
key_id: str
secret_key: str
@dataclass
class GarageNode:
node_id: str
host: str
def get_node_fqn(machine: Machine) -> GarageNode:
node_id, host = machine.succeed("garage node id").split('@')
return GarageNode(node_id=node_id, host=host)
def get_node_id(machine: Machine) -> str:
return get_node_fqn(machine).node_id
def get_layout_version(machine: Machine) -> int:
version_data = machine.succeed("garage layout show")
m = cur_version_regex.search(version_data)
if m and m.group('ver') is not None:
return int(m.group('ver')) + 1
else:
raise ValueError('Cannot find current layout version')
def apply_garage_layout(machine: Machine, layouts: List[str]):
for layout in layouts:
machine.succeed(f"garage layout assign {layout}")
version = get_layout_version(machine)
machine.succeed(f"garage layout apply --version {version}")
def create_api_key(machine: Machine, key_name: str) -> S3Key:
output = machine.succeed(f"garage key create {key_name}")
return parse_api_key_data(output)
def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
output = machine.succeed(f"garage key info {key_pattern}")
return parse_api_key_data(output)
def parse_api_key_data(text) -> S3Key:
key_creation_regex = re.compile(r'Key name: \s*(?P<key_name>.*)|' r'Key ID: \s*(?P<key_id>.*)|' r'Secret key: \s*(?P<secret_key>.*)', re.IGNORECASE)
fields = {}
for match in key_creation_regex.finditer(text):
for key, value in match.groupdict().items():
if value:
fields[key] = value.strip()
try:
return S3Key(**fields)
except TypeError as e:
raise ValueError(f"Cannot parse API key data. Missing required field(s): {e}")
def test_bucket_writes(node):
node.succeed("garage bucket create test-bucket")
s3_key = create_api_key(node, "test-api-key")
node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
other_s3_key = get_api_key(node, 'test-api-key')
assert other_s3_key.secret_key == other_s3_key.secret_key
node.succeed(
f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
)
node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")
assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"
def test_bucket_over_http(node, bucket='test-bucket', url=None):
if url is None:
url = f"{bucket}.web.garage"
node.succeed(f'garage bucket website --allow {bucket}')
node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')
assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
'';
}
nixos/tests/garage/default.nix
+25 -31
View File
@@ -1,16 +1,12 @@
{
system ? builtins.currentSystem,
config ? { },
pkgs ? import ../../.. { inherit system config; },
runTest,
package,
}:
with pkgs.lib;
let
mkNode =
package:
{
replicationMode,
publicV6Address ? "::1",
extraSettings ? { },
}:
{ pkgs, ... }:
{
@@ -30,8 +26,6 @@ let
enable = true;
inherit package;
settings = {
replication_mode = replicationMode;
rpc_bind_addr = "[::]:3901";
rpc_public_addr = "[${publicV6Address}]:3901";
rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c";
@@ -47,7 +41,7 @@ let
root_domain = ".web.garage";
index = "index.html";
};
};
} // extraSettings;
};
environment.systemPackages = [ pkgs.minio-client ];
@@ -55,24 +49,24 @@ let
virtualisation.diskSize = 2 * 1024;
};
in
foldl
(
matrix: ver:
matrix
// {
"basic${toString ver}" = import ./basic.nix {
inherit system pkgs ver;
mkNode = mkNode pkgs."garage_${ver}";
};
"with-3node-replication${toString ver}" = import ./with-3node-replication.nix {
inherit system pkgs ver;
mkNode = mkNode pkgs."garage_${ver}";
};
}
)
{ }
[
"0_8"
"0_9"
"1_x"
]
{
basic = runTest {
imports = [
./common.nix
./basic.nix
];
_module.args = {
inherit mkNode package;
};
};
with-3node-replication = runTest {
imports = [
./common.nix
./with-3node-replication.nix
];
_module.args = {
inherit mkNode package;
};
};
}
nixos/tests/garage/with-3node-replication.nix
+43 -105
View File
@@ -1,107 +1,47 @@
args@{ mkNode, ver, ... }:
(import ../make-test-python.nix (
{ pkgs, ... }:
{
name = "garage-3node-replication";
meta = {
maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
{
lib,
mkNode,
package,
testScriptSetup,
...
}:
let
extraSettings =
if (lib.versionAtLeast package.version "2") then
{
replication_factor = 3;
consistency_mode = "consistent";
}
else
{
replication_mode = "3";
};
in
{
name = "garage-3node-replication";
nodes = {
node1 = mkNode {
inherit extraSettings;
publicV6Address = "fc00:1::1";
};
nodes = {
node1 = mkNode {
replicationMode = "3";
publicV6Address = "fc00:1::1";
};
node2 = mkNode {
replicationMode = "3";
publicV6Address = "fc00:1::2";
};
node3 = mkNode {
replicationMode = "3";
publicV6Address = "fc00:1::3";
};
node4 = mkNode {
replicationMode = "3";
publicV6Address = "fc00:1::4";
};
node2 = mkNode {
inherit extraSettings;
publicV6Address = "fc00:1::2";
};
node3 = mkNode {
inherit extraSettings;
publicV6Address = "fc00:1::3";
};
node4 = mkNode {
inherit extraSettings;
publicV6Address = "fc00:1::4";
};
};
testScript = ''
from typing import List
from dataclasses import dataclass
import re
start_all()
cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')
key_creation_regex = re.compile('Key name: (?P<key_name>.*)\nKey ID: (?P<key_id>.*)\nSecret key: (?P<secret_key>.*)')
@dataclass
class S3Key:
key_name: str
key_id: str
secret_key: str
@dataclass
class GarageNode:
node_id: str
host: str
def get_node_fqn(machine: Machine) -> GarageNode:
node_id, host = machine.succeed("garage node id").split('@')
return GarageNode(node_id=node_id, host=host)
def get_node_id(machine: Machine) -> str:
return get_node_fqn(machine).node_id
def get_layout_version(machine: Machine) -> int:
version_data = machine.succeed("garage layout show")
m = cur_version_regex.search(version_data)
if m and m.group('ver') is not None:
return int(m.group('ver')) + 1
else:
raise ValueError('Cannot find current layout version')
def apply_garage_layout(machine: Machine, layouts: List[str]):
for layout in layouts:
machine.succeed(f"garage layout assign {layout}")
version = get_layout_version(machine)
machine.succeed(f"garage layout apply --version {version}")
def create_api_key(machine: Machine, key_name: str) -> S3Key:
output = machine.succeed(f"garage key ${
if ver == "0_8" then "new --name" else "create"
} {key_name}")
m = key_creation_regex.match(output)
if not m or not m.group('key_id') or not m.group('secret_key'):
raise ValueError('Cannot parse API key data')
return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))
def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
output = machine.succeed(f"garage key info {key_pattern}")
m = key_creation_regex.match(output)
if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):
raise ValueError('Cannot parse API key data')
return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))
def test_bucket_writes(node):
node.succeed("garage bucket create test-bucket")
s3_key = create_api_key(node, "test-api-key")
node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
other_s3_key = get_api_key(node, 'test-api-key')
assert other_s3_key.secret_key == other_s3_key.secret_key
node.succeed(
f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
)
node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")
assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"
def test_bucket_over_http(node, bucket='test-bucket', url=None):
if url is None:
url = f"{bucket}.web.garage"
node.succeed(f'garage bucket website --allow {bucket}')
node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')
assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
testScript = # python
''
${testScriptSetup}
with subtest("Garage works as a multi-node S3 storage"):
nodes = ('node1', 'node2', 'node3', 'node4')
@@ -125,7 +65,7 @@ args@{ mkNode, ver, ... }:
zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"]
apply_garage_layout(node1,
[
f'{ndata.node_id} -z {zones[index]} -c ${if ver == "0_8" then "1" else "1G"}'
f'{ndata.node_id} -z {zones[index]} -c 1G'
for index, ndata in enumerate(node_ids.values())
])
# Now Garage is operational.
@@ -133,6 +73,4 @@ args@{ mkNode, ver, ... }:
for node in nodes:
test_bucket_over_http(get_machine(node))
'';
}
))
args
}
pkgs/tools/filesystems/garage/default.nix
+10 -1
View File
@@ -92,7 +92,7 @@ let
"k2v::poll::test_poll_item"
];
passthru.tests = nixosTests.garage;
passthru.tests = nixosTests."garage_${lib.versions.major version}";
meta = {
description = "S3-compatible object store for small self-hosted geo-distributed deployments";
@@ -137,11 +137,20 @@ rec {
cargoHash = "sha256-vcvD0Fn/etnAuXrM3+rj16cqpEmW2nzRmrjXsftKTFE=";
};
garage_2_0_0 = generic {
version = "2.0.0";
hash = "sha256-dn7FoouF+5qmW6fcC20bKQSc6D2G9yrWdBK3uN3bF58=";
cargoHash = "sha256-6VM/EesrUIaQOeDGqzb0kOqMz4hW7zBJUnaRQ9C3cqc=";
};
garage_0_8 = garage_0_8_7;
garage_0_9 = garage_0_9_4;
garage_1_x = garage_1_2_0;
garage_1 = garage_1_x;
garage_2 = garage_2_0_0;
garage = garage_1_x;
}
pkgs/top-level/all-packages.nix
+5
View File
@@ -3029,8 +3029,13 @@ with pkgs;
garage_0_9
garage_0_8_7
garage_0_9_4
garage_1_2_0
garage_1_x
garage_1
garage_2_0_0
garage_2
;
gaugePlugins = recurseIntoAttrs (callPackage ../by-name/ga/gauge/plugins { });