garage_2: init at 2.0.0, move tests to runTest (#417549) (0e6feffe) · Commits · nix / nixpkgs

nixos/tests/all-tests.nix

+8 −1

Original line number	Diff line number	Diff line
		@@ -543,7 +543,14 @@ in
		mimir = runTest ./mimir.nix;
		galene = discoverTests (import ./galene.nix);
		gancio = runTest ./gancio.nix;
		garage = handleTest ./garage { };
		garage_1 = import ./garage {
		inherit runTest;
		package = pkgs.garage_1;
		};
		garage_2 = import ./garage {
		inherit runTest;
		package = pkgs.garage_2;
		};
		gatus = runTest ./gatus.nix;
		getaddrinfo = runTest ./getaddrinfo.nix;
		gemstash = handleTest ./gemstash.nix { };

nixos/tests/garage/basic.nix

+28 −94

Original line number	Diff line number	Diff line
		args@{ mkNode, ver, ... }:
		(import ../make-test-python.nix (
		{ pkgs, ... }:
		{
		lib,
		mkNode,
		package,
		testScriptSetup,
		...
		}:
		{
		name = "garage-basic";
		meta = {
		maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
		};

		nodes = {
		single_node = mkNode { replicationMode = "none"; };
		single_node = mkNode {
		extraSettings =
		if (lib.versionAtLeast package.version "2") then
		{
		replication_factor = 1;
		consistency_mode = "consistent";
		}
		else
		{
		replication_mode = "none";
		};
		};
		};

		testScript = ''
		from typing import List
		from dataclasses import dataclass
		import re

		start_all()

		cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')
		key_creation_regex = re.compile('Key name: (?P<key_name>.)\nKey ID: (?P<key_id>.)\nSecret key: (?P<secret_key>.*)')

		@dataclass
		class S3Key:
		key_name: str
		key_id: str
		secret_key: str

		@dataclass
		class GarageNode:
		node_id: str
		host: str

		def get_node_fqn(machine: Machine) -> GarageNode:
		node_id, host = machine.succeed("garage node id").split('@')
		return GarageNode(node_id=node_id, host=host)

		def get_node_id(machine: Machine) -> str:
		return get_node_fqn(machine).node_id

		def get_layout_version(machine: Machine) -> int:
		version_data = machine.succeed("garage layout show")
		m = cur_version_regex.search(version_data)
		if m and m.group('ver') is not None:
		return int(m.group('ver')) + 1
		else:
		raise ValueError('Cannot find current layout version')

		def apply_garage_layout(machine: Machine, layouts: List[str]):
		for layout in layouts:
		machine.succeed(f"garage layout assign {layout}")
		version = get_layout_version(machine)
		machine.succeed(f"garage layout apply --version {version}")

		def create_api_key(machine: Machine, key_name: str) -> S3Key:
		output = machine.succeed(f"garage key ${
		if ver == "0_8" then "new --name" else "create"
		} {key_name}")
		m = key_creation_regex.match(output)
		if not m or not m.group('key_id') or not m.group('secret_key'):
		raise ValueError('Cannot parse API key data')
		return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))

		def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
		output = machine.succeed(f"garage key info {key_pattern}")
		m = key_creation_regex.match(output)
		if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):
		raise ValueError('Cannot parse API key data')
		return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))

		def test_bucket_writes(node):
		node.succeed("garage bucket create test-bucket")
		s3_key = create_api_key(node, "test-api-key")
		node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
		other_s3_key = get_api_key(node, 'test-api-key')
		assert other_s3_key.secret_key == other_s3_key.secret_key
		node.succeed(
		f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
		)
		node.succeed("echo test \| mc pipe test-garage/test-bucket/test.txt")
		assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"

		def test_bucket_over_http(node, bucket='test-bucket', url=None):
		if url is None:
		url = f"{bucket}.web.garage"

		node.succeed(f'garage bucket website --allow {bucket}')
		node.succeed(f'echo hello world \| mc pipe test-garage/{bucket}/index.html')
		assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
		testScript = # python
		''
		${testScriptSetup}

		with subtest("Garage works as a single-node S3 storage"):
		single_node.wait_for_unit("garage.service")
		single_node.wait_for_open_port(3900)
		# Now Garage is initialized.
		single_node_id = get_node_id(single_node)
		apply_garage_layout(single_node, [f'-z qemutest -c ${
		if ver == "0_8" then "1" else "1G"
		} "{single_node_id}"'])
		apply_garage_layout(single_node, [f'-z qemutest -c 1G "{single_node_id}"'])
		# Now Garage is operational.
		test_bucket_writes(single_node)
		test_bucket_over_http(single_node)
		'';
		}
		))
		args

nixos/tests/garage/common.nix

0 → 100644

+85 −0

Original line number	Diff line number	Diff line
		{ ... }:
		{
		_module.args.testScriptSetup = # python
		''
		from typing import List
		from dataclasses import dataclass
		import re

		start_all()

		cur_version_regex = re.compile(r'Current cluster layout version: (?P<ver>\d*)')

		@dataclass
		class S3Key:
		key_name: str
		key_id: str
		secret_key: str

		@dataclass
		class GarageNode:
		node_id: str
		host: str

		def get_node_fqn(machine: Machine) -> GarageNode:
		node_id, host = machine.succeed("garage node id").split('@')
		return GarageNode(node_id=node_id, host=host)

		def get_node_id(machine: Machine) -> str:
		return get_node_fqn(machine).node_id

		def get_layout_version(machine: Machine) -> int:
		version_data = machine.succeed("garage layout show")
		m = cur_version_regex.search(version_data)
		if m and m.group('ver') is not None:
		return int(m.group('ver')) + 1
		else:
		raise ValueError('Cannot find current layout version')

		def apply_garage_layout(machine: Machine, layouts: List[str]):
		for layout in layouts:
		machine.succeed(f"garage layout assign {layout}")
		version = get_layout_version(machine)
		machine.succeed(f"garage layout apply --version {version}")

		def create_api_key(machine: Machine, key_name: str) -> S3Key:
		output = machine.succeed(f"garage key create {key_name}")
		return parse_api_key_data(output)

		def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
		output = machine.succeed(f"garage key info {key_pattern}")
		return parse_api_key_data(output)

		def parse_api_key_data(text) -> S3Key:
		key_creation_regex = re.compile(r'Key name: \s(?P<key_name>.)\|' r'Key ID: \s(?P<key_id>.)\|' r'Secret key: \s(?P<secret_key>.)', re.IGNORECASE)
		fields = {}
		for match in key_creation_regex.finditer(text):
		for key, value in match.groupdict().items():
		if value:
		fields[key] = value.strip()
		try:
		return S3Key(**fields)
		except TypeError as e:
		raise ValueError(f"Cannot parse API key data. Missing required field(s): {e}")

		def test_bucket_writes(node):
		node.succeed("garage bucket create test-bucket")
		s3_key = create_api_key(node, "test-api-key")
		node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
		other_s3_key = get_api_key(node, 'test-api-key')
		assert other_s3_key.secret_key == other_s3_key.secret_key
		node.succeed(
		f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
		)
		node.succeed("echo test \| mc pipe test-garage/test-bucket/test.txt")
		assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"

		def test_bucket_over_http(node, bucket='test-bucket', url=None):
		if url is None:
		url = f"{bucket}.web.garage"

		node.succeed(f'garage bucket website --allow {bucket}')
		node.succeed(f'echo hello world \| mc pipe test-garage/{bucket}/index.html')
		assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
		'';
		}

nixos/tests/garage/default.nix

+25 −31

Original line number	Diff line number	Diff line
		{
		system ? builtins.currentSystem,
		config ? { },
		pkgs ? import ../../.. { inherit system config; },
		runTest,
		package,
		}:
		with pkgs.lib;

		let
		mkNode =
		package:
		{
		replicationMode,
		publicV6Address ? "::1",
		extraSettings ? { },
		}:
		{ pkgs, ... }:
		{
		@@ -30,8 +26,6 @@ let
		enable = true;
		inherit package;
		settings = {
		replication_mode = replicationMode;

		rpc_bind_addr = "[::]:3901";
		rpc_public_addr = "[${publicV6Address}]:3901";
		rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c";
		@@ -47,7 +41,7 @@ let
		root_domain = ".web.garage";
		index = "index.html";
		};
		};
		} // extraSettings;
		};
		environment.systemPackages = [ pkgs.minio-client ];

		@@ -55,24 +49,24 @@ let
		virtualisation.diskSize = 2 * 1024;
		};
		in
		foldl
		(
		matrix: ver:
		matrix
		// {
		"basic${toString ver}" = import ./basic.nix {
		inherit system pkgs ver;
		mkNode = mkNode pkgs."garage_${ver}";
		{
		basic = runTest {
		imports = [
		./common.nix
		./basic.nix
		];
		_module.args = {
		inherit mkNode package;
		};
		};

		with-3node-replication = runTest {
		imports = [
		./common.nix
		./with-3node-replication.nix
		];
		_module.args = {
		inherit mkNode package;
		};
		"with-3node-replication${toString ver}" = import ./with-3node-replication.nix {
		inherit system pkgs ver;
		mkNode = mkNode pkgs."garage_${ver}";
		};
		}
		)
		{ }
		[
		"0_8"
		"0_9"
		"1_x"
		]

nixos/tests/garage/with-3node-replication.nix

+43 −105

Original line number	Diff line number	Diff line
		args@{ mkNode, ver, ... }:
		(import ../make-test-python.nix (
		{ pkgs, ... }:
		{
		name = "garage-3node-replication";
		meta = {
		maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
		lib,
		mkNode,
		package,
		testScriptSetup,
		...
		}:
		let
		extraSettings =
		if (lib.versionAtLeast package.version "2") then
		{
		replication_factor = 3;
		consistency_mode = "consistent";
		}
		else
		{
		replication_mode = "3";
		};
		in
		{
		name = "garage-3node-replication";

		nodes = {
		node1 = mkNode {
		replicationMode = "3";
		inherit extraSettings;
		publicV6Address = "fc00:1::1";
		};
		node2 = mkNode {
		replicationMode = "3";
		inherit extraSettings;
		publicV6Address = "fc00:1::2";
		};
		node3 = mkNode {
		replicationMode = "3";
		inherit extraSettings;
		publicV6Address = "fc00:1::3";
		};
		node4 = mkNode {
		replicationMode = "3";
		inherit extraSettings;
		publicV6Address = "fc00:1::4";
		};
		};

		testScript = ''
		from typing import List
		from dataclasses import dataclass
		import re
		start_all()

		cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')
		key_creation_regex = re.compile('Key name: (?P<key_name>.)\nKey ID: (?P<key_id>.)\nSecret key: (?P<secret_key>.*)')

		@dataclass
		class S3Key:
		key_name: str
		key_id: str
		secret_key: str

		@dataclass
		class GarageNode:
		node_id: str
		host: str

		def get_node_fqn(machine: Machine) -> GarageNode:
		node_id, host = machine.succeed("garage node id").split('@')
		return GarageNode(node_id=node_id, host=host)

		def get_node_id(machine: Machine) -> str:
		return get_node_fqn(machine).node_id

		def get_layout_version(machine: Machine) -> int:
		version_data = machine.succeed("garage layout show")
		m = cur_version_regex.search(version_data)
		if m and m.group('ver') is not None:
		return int(m.group('ver')) + 1
		else:
		raise ValueError('Cannot find current layout version')

		def apply_garage_layout(machine: Machine, layouts: List[str]):
		for layout in layouts:
		machine.succeed(f"garage layout assign {layout}")
		version = get_layout_version(machine)
		machine.succeed(f"garage layout apply --version {version}")

		def create_api_key(machine: Machine, key_name: str) -> S3Key:
		output = machine.succeed(f"garage key ${
		if ver == "0_8" then "new --name" else "create"
		} {key_name}")
		m = key_creation_regex.match(output)
		if not m or not m.group('key_id') or not m.group('secret_key'):
		raise ValueError('Cannot parse API key data')
		return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))

		def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
		output = machine.succeed(f"garage key info {key_pattern}")
		m = key_creation_regex.match(output)
		if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):
		raise ValueError('Cannot parse API key data')
		return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))

		def test_bucket_writes(node):
		node.succeed("garage bucket create test-bucket")
		s3_key = create_api_key(node, "test-api-key")
		node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
		other_s3_key = get_api_key(node, 'test-api-key')
		assert other_s3_key.secret_key == other_s3_key.secret_key
		node.succeed(
		f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
		)
		node.succeed("echo test \| mc pipe test-garage/test-bucket/test.txt")
		assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"

		def test_bucket_over_http(node, bucket='test-bucket', url=None):
		if url is None:
		url = f"{bucket}.web.garage"

		node.succeed(f'garage bucket website --allow {bucket}')
		node.succeed(f'echo hello world \| mc pipe test-garage/{bucket}/index.html')
		assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
		testScript = # python
		''
		${testScriptSetup}

		with subtest("Garage works as a multi-node S3 storage"):
		nodes = ('node1', 'node2', 'node3', 'node4')
		@@ -125,7 +65,7 @@ args@{ mkNode, ver, ... }:
		zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"]
		apply_garage_layout(node1,
		[
		f'{ndata.node_id} -z {zones[index]} -c ${if ver == "0_8" then "1" else "1G"}'
		f'{ndata.node_id} -z {zones[index]} -c 1G'
		for index, ndata in enumerate(node_ids.values())
		])
		# Now Garage is operational.
		@@ -134,5 +74,3 @@ args@{ mkNode, ver, ... }:
		test_bucket_over_http(get_machine(node))
		'';
		}
		))
		args

Admin message