Unverified Commit b9700f76 authored by Adam C. Stephens's avatar Adam C. Stephens
Browse files

garage_2: init at 2.0.0 

https://garagehq.deuxfleurs.fr/blog/2025-06-garage-v2/
https://git.deuxfleurs.fr/Deuxfleurs/garage/releases/tag/v2.0.0

Thanks to @herbetom for providing the test updates
parent 72260747

nixos/tests/all-tests.nix

+5 −1
Original line number Diff line number Diff line
@@ -537,7 +537,11 @@ in 
  gancio = runTest ./gancio.nix;

  garage_1 = import ./garage {

    inherit runTest;

    package = pkgs.garage_1_x;

    package = pkgs.garage_1;

  };

  garage_2 = import ./garage {

    inherit runTest;

    package = pkgs.garage_2;

  };

  gatus = runTest ./gatus.nix;

  getaddrinfo = runTest ./getaddrinfo.nix;

nixos/tests/garage/basic.nix

+20 −76
Original line number Diff line number Diff line 
{ mkNode, ... }:

{

  lib,

  mkNode,

  package,

  testScriptSetup,

  ...

}:

{

  name = "garage-basic";



  nodes = {

    single_node = mkNode { replicationMode = "none"; };

    single_node = mkNode {

      extraSettings =

        if (lib.versionAtLeast package.version "2") then

          {

            replication_factor = 1;

            consistency_mode = "consistent";

          }

        else

          {

            replication_mode = "none";

          };

    };

  };



  testScript = # python

    ''

      from typing import List

      from dataclasses import dataclass

      import re



      start_all()



      cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')

      key_creation_regex = re.compile('Key name: (?P<key_name>.*)\nKey ID: (?P<key_id>.*)\nSecret key: (?P<secret_key>.*)')



      @dataclass

      class S3Key:

         key_name: str

         key_id: str

         secret_key: str



      @dataclass

      class GarageNode:

         node_id: str

         host: str



      def get_node_fqn(machine: Machine) -> GarageNode:

        node_id, host = machine.succeed("garage node id").split('@')

        return GarageNode(node_id=node_id, host=host)



      def get_node_id(machine: Machine) -> str:

        return get_node_fqn(machine).node_id



      def get_layout_version(machine: Machine) -> int:

        version_data = machine.succeed("garage layout show")

        m = cur_version_regex.search(version_data)

        if m and m.group('ver') is not None:

          return int(m.group('ver')) + 1

        else:

          raise ValueError('Cannot find current layout version')



      def apply_garage_layout(machine: Machine, layouts: List[str]):

         for layout in layouts:

            machine.succeed(f"garage layout assign {layout}")

         version = get_layout_version(machine)

         machine.succeed(f"garage layout apply --version {version}")



      def create_api_key(machine: Machine, key_name: str) -> S3Key:

         output = machine.succeed(f"garage key create {key_name}")

         m = key_creation_regex.match(output)

         if not m or not m.group('key_id') or not m.group('secret_key'):

            raise ValueError('Cannot parse API key data')

         return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))



      def get_api_key(machine: Machine, key_pattern: str) -> S3Key:

         output = machine.succeed(f"garage key info {key_pattern}")

         m = key_creation_regex.match(output)

         if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):

             raise ValueError('Cannot parse API key data')

         return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))



      def test_bucket_writes(node):

        node.succeed("garage bucket create test-bucket")

        s3_key = create_api_key(node, "test-api-key")

        node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")

        other_s3_key = get_api_key(node, 'test-api-key')

        assert other_s3_key.secret_key == other_s3_key.secret_key

        node.succeed(

          f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"

        )

        node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")

        assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"



      def test_bucket_over_http(node, bucket='test-bucket', url=None):

        if url is None:

           url = f"{bucket}.web.garage"



        node.succeed(f'garage bucket website --allow {bucket}')

        node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')

        assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'

      ${testScriptSetup}



      with subtest("Garage works as a single-node S3 storage"):

        single_node.wait_for_unit("garage.service")

nixos/tests/garage/common.nix

0 → 100644
+85 −0
Original line number Diff line number Diff line 
{ ... }:

{

  _module.args.testScriptSetup = # python

    ''

      from typing import List

      from dataclasses import dataclass

      import re



      start_all()



      cur_version_regex = re.compile(r'Current cluster layout version: (?P<ver>\d*)')



      @dataclass

      class S3Key:

         key_name: str

         key_id: str

         secret_key: str



      @dataclass

      class GarageNode:

         node_id: str

         host: str



      def get_node_fqn(machine: Machine) -> GarageNode:

        node_id, host = machine.succeed("garage node id").split('@')

        return GarageNode(node_id=node_id, host=host)



      def get_node_id(machine: Machine) -> str:

        return get_node_fqn(machine).node_id



      def get_layout_version(machine: Machine) -> int:

        version_data = machine.succeed("garage layout show")

        m = cur_version_regex.search(version_data)

        if m and m.group('ver') is not None:

          return int(m.group('ver')) + 1

        else:

          raise ValueError('Cannot find current layout version')



      def apply_garage_layout(machine: Machine, layouts: List[str]):

         for layout in layouts:

            machine.succeed(f"garage layout assign {layout}")

         version = get_layout_version(machine)

         machine.succeed(f"garage layout apply --version {version}")



      def create_api_key(machine: Machine, key_name: str) -> S3Key:

         output = machine.succeed(f"garage key create {key_name}")

         return parse_api_key_data(output)



      def get_api_key(machine: Machine, key_pattern: str) -> S3Key:

         output = machine.succeed(f"garage key info {key_pattern}")

         return parse_api_key_data(output)



      def parse_api_key_data(text) -> S3Key:

        key_creation_regex = re.compile(r'Key name: \s*(?P<key_name>.*)|' r'Key ID: \s*(?P<key_id>.*)|' r'Secret key: \s*(?P<secret_key>.*)', re.IGNORECASE)

        fields = {}

        for match in key_creation_regex.finditer(text):

          for key, value in match.groupdict().items():

            if value:

              fields[key] = value.strip()

        try:

          return S3Key(**fields)

        except TypeError as e:

          raise ValueError(f"Cannot parse API key data. Missing required field(s): {e}")



      def test_bucket_writes(node):

        node.succeed("garage bucket create test-bucket")

        s3_key = create_api_key(node, "test-api-key")

        node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")

        other_s3_key = get_api_key(node, 'test-api-key')

        assert other_s3_key.secret_key == other_s3_key.secret_key

        node.succeed(

          f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"

        )

        node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")

        assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"



      def test_bucket_over_http(node, bucket='test-bucket', url=None):

        if url is None:

           url = f"{bucket}.web.garage"



        node.succeed(f'garage bucket website --allow {bucket}')

        node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')

        assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'

    '';

}

nixos/tests/garage/default.nix

+6 −6
Original line number Diff line number Diff line
@@ -5,8 +5,8 @@ 
let

  mkNode =

    {

      replicationMode,

      publicV6Address ? "::1",

      extraSettings ? { },

    }:

    { pkgs, ... }:

    {
@@ -26,8 +26,6 @@ let 
        enable = true;

        inherit package;

        settings = {

          replication_mode = replicationMode;



          rpc_bind_addr = "[::]:3901";

          rpc_public_addr = "[${publicV6Address}]:3901";

          rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c";
@@ -43,7 +41,7 @@ let 
            root_domain = ".web.garage";

            index = "index.html";

          };

        };

        } // extraSettings;

      };

      environment.systemPackages = [ pkgs.minio-client ];
@@ -54,19 +52,21 @@ in 
{

  basic = runTest {

    imports = [

      ./common.nix

      ./basic.nix

    ];

    _module.args = {

      inherit mkNode;

      inherit mkNode package;

    };

  };



  with-3node-replication = runTest {

    imports = [

      ./common.nix

      ./with-3node-replication.nix

    ];

    _module.args = {

      inherit mkNode;

      inherit mkNode package;

    };

  };

}

nixos/tests/garage/with-3node-replication.nix

+24 −78
Original line number Diff line number Diff line 
{ mkNode, ... }:

{

  lib,

  mkNode,

  package,

  testScriptSetup,

  ...

}:

let

  extraSettings =

    if (lib.versionAtLeast package.version "2") then

      {

        replication_factor = 3;

        consistency_mode = "consistent";

      }

    else

      {

        replication_mode = "3";

      };

in

{

  name = "garage-3node-replication";



  nodes = {

    node1 = mkNode {

      replicationMode = "3";

      inherit extraSettings;

      publicV6Address = "fc00:1::1";

    };

    node2 = mkNode {

      replicationMode = "3";

      inherit extraSettings;

      publicV6Address = "fc00:1::2";

    };

    node3 = mkNode {

      replicationMode = "3";

      inherit extraSettings;

      publicV6Address = "fc00:1::3";

    };

    node4 = mkNode {

      replicationMode = "3";

      inherit extraSettings;

      publicV6Address = "fc00:1::4";

    };

  };



  testScript = # python

    ''

      from typing import List

      from dataclasses import dataclass

      import re

      start_all()



      cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')

      key_creation_regex = re.compile('Key name: (?P<key_name>.*)\nKey ID: (?P<key_id>.*)\nSecret key: (?P<secret_key>.*)')



      @dataclass

      class S3Key:

         key_name: str

         key_id: str

         secret_key: str



      @dataclass

      class GarageNode:

         node_id: str

         host: str



      def get_node_fqn(machine: Machine) -> GarageNode:

        node_id, host = machine.succeed("garage node id").split('@')

        return GarageNode(node_id=node_id, host=host)



      def get_node_id(machine: Machine) -> str:

        return get_node_fqn(machine).node_id



      def get_layout_version(machine: Machine) -> int:

        version_data = machine.succeed("garage layout show")

        m = cur_version_regex.search(version_data)

        if m and m.group('ver') is not None:

          return int(m.group('ver')) + 1

        else:

          raise ValueError('Cannot find current layout version')



      def apply_garage_layout(machine: Machine, layouts: List[str]):

         for layout in layouts:

            machine.succeed(f"garage layout assign {layout}")

         version = get_layout_version(machine)

         machine.succeed(f"garage layout apply --version {version}")



      def create_api_key(machine: Machine, key_name: str) -> S3Key:

         output = machine.succeed(f"garage key create {key_name}")

         m = key_creation_regex.match(output)

         if not m or not m.group('key_id') or not m.group('secret_key'):

            raise ValueError('Cannot parse API key data')

         return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))



      def get_api_key(machine: Machine, key_pattern: str) -> S3Key:

         output = machine.succeed(f"garage key info {key_pattern}")

         m = key_creation_regex.match(output)

         if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):

             raise ValueError('Cannot parse API key data')

         return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))



      def test_bucket_writes(node):

        node.succeed("garage bucket create test-bucket")

        s3_key = create_api_key(node, "test-api-key")

        node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")

        other_s3_key = get_api_key(node, 'test-api-key')

        assert other_s3_key.secret_key == other_s3_key.secret_key

        node.succeed(

          f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"

        )

        node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")

        assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"



      def test_bucket_over_http(node, bucket='test-bucket', url=None):

        if url is None:

           url = f"{bucket}.web.garage"



        node.succeed(f'garage bucket website --allow {bucket}')

        node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')

        assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'

      ${testScriptSetup}



      with subtest("Garage works as a multi-node S3 storage"):

        nodes = ('node1', 'node2', 'node3', 'node4')