ceph: Fix missing patch for Ceph with dmcrypt (#335684) (53152af2) · Commits · nix / nixpkgs

nixos/tests/all-tests.nix

+1 −0

Original line number	Diff line number	Diff line
		@@ -188,6 +188,7 @@ in {
		ceph-multi-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-multi-node.nix {};
		ceph-single-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node.nix {};
		ceph-single-node-bluestore = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node-bluestore.nix {};
		ceph-single-node-bluestore-dmcrypt = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node-bluestore-dmcrypt.nix {};
		certmgr = handleTest ./certmgr.nix {};
		cfssl = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cfssl.nix {};
		cgit = handleTest ./cgit.nix {};

nixos/tests/ceph-single-node-bluestore-dmcrypt.nix

0 → 100644

+270 −0

Original line number	Diff line number	Diff line
		import ./make-test-python.nix (
		{ pkgs, lib, ... }:

		let
		# the single node ipv6 address
		ip = "2001:db8:ffff::";
		# the global ceph cluster id
		cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a";
		# the fsids of OSDs
		osd-fsid-map = {
		"0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa";
		"1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c";
		"2" = "c90de6c7-86c6-41da-9694-e794096dfc5c";
		};

		in
		{
		name = "basic-single-node-ceph-cluster-bluestore-dmcrypt";
		meta = with pkgs.lib.maintainers; {
		maintainers = [ benaryorg nh2 ];
		};

		nodes = {
		ceph =
		{ pkgs, config, ... }:
		{
		# disks for bluestore
		virtualisation.emptyDiskImages = [
		20480
		20480
		20480
		];

		# networking setup (no external connectivity required, only local IPv6)
		networking.useDHCP = false;
		systemd.network = {
		enable = true;
		wait-online.extraArgs = [
		"-i"
		"lo"
		];
		networks = {
		"40-loopback" = {
		enable = true;
		name = "lo";
		DHCP = "no";
		addresses = [ { Address = "${ip}/128"; } ];
		};
		};
		};

		# do not start the ceph target by default so we can format the disks first
		systemd.targets.ceph.wantedBy = lib.mkForce [ ];

		# add the packages to systemPackages so the testscript doesn't run into any unexpected issues
		# this shouldn't be required on production systems which have their required packages in the unit paths only
		# but it helps in case one needs to actually run the tooling anyway
		environment.systemPackages = with pkgs; [
		ceph
		cryptsetup
		lvm2
		];

		services.ceph = {
		enable = true;
		client.enable = true;
		extraConfig = {
		public_addr = ip;
		cluster_addr = ip;
		# ipv6
		ms_bind_ipv4 = "false";
		ms_bind_ipv6 = "true";
		# msgr2 settings
		ms_cluster_mode = "secure";
		ms_service_mode = "secure";
		ms_client_mode = "secure";
		ms_mon_cluster_mode = "secure";
		ms_mon_service_mode = "secure";
		ms_mon_client_mode = "secure";
		# less default modules, cuts down on memory and startup time in the tests
		mgr_initial_modules = "";
		# distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host
		osd_crush_chooseleaf_type = "0";
		};
		client.extraConfig."mon.0" = {
		host = "ceph";
		mon_addr = "v2:[${ip}]:3300";
		public_addr = "v2:[${ip}]:3300";
		};
		global = {
		fsid = cluster;
		clusterNetwork = "${ip}/64";
		publicNetwork = "${ip}/64";
		monInitialMembers = "0";
		};

		mon = {
		enable = true;
		daemons = [ "0" ];
		};

		osd = {
		enable = true;
		daemons = builtins.attrNames osd-fsid-map;
		};

		mgr = {
		enable = true;
		daemons = [ "ceph" ];
		};
		};

		systemd.services =
		let
		osd-name = id: "ceph-osd-${id}";
		osd-pre-start = id: [
		"!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd"
		"${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}"
		];
		osd-post-stop = id: [
		"!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}"
		];
		map-osd = id: {
		name = osd-name id;
		value = {
		serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id);
		serviceConfig.ExecStopPost = osd-post-stop id;
		unitConfig.ConditionPathExists = lib.mkForce [ ];
		unitConfig.StartLimitBurst = lib.mkForce 4;
		path = with pkgs; [
		util-linux
		lvm2
		cryptsetup
		];
		};
		};
		in
		lib.pipe config.services.ceph.osd.daemons [
		(builtins.map map-osd)
		builtins.listToAttrs
		];
		};
		};

		testScript =
		{ ... }:
		''
		start_all()

		ceph.wait_for_unit("default.target")

		# Bootstrap ceph-mon daemon
		ceph.succeed(
		"mkdir -p /var/lib/ceph/bootstrap-osd",
		"ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
		"ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow ' --cap osd 'allow ' --cap mds 'allow ' --cap mgr 'allow '",
		"ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'",
		"ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
		"ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
		"monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap",
		"mkdir -p /var/lib/ceph/mon/ceph-0",
		"ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring",
		"chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph",
		"systemctl start ceph-mon-0.service",
		)

		ceph.wait_for_unit("ceph-mon-0.service")
		# should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout
		ceph.wait_for_open_port(3300, "${ip}")
		ceph.succeed(
		# required for HEALTH_OK
		"ceph config set mon auth_allow_insecure_global_id_reclaim false",
		# IPv6
		"ceph config set global ms_bind_ipv4 false",
		"ceph config set global ms_bind_ipv6 true",
		# the new (secure) protocol
		"ceph config set global ms_bind_msgr1 false",
		"ceph config set global ms_bind_msgr2 true",
		# just a small little thing
		"ceph config set mon mon_compact_on_start true",
		)

		# Can't check ceph status until a mon is up
		ceph.succeed("ceph -s \| grep 'mon: 1 daemons'")

		# Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory)
		ceph.succeed(
		# this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases
		"ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}",
		"ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}",
		"ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}",
		"sudo ceph-volume lvm deactivate 0",
		"sudo ceph-volume lvm deactivate 1",
		"sudo ceph-volume lvm deactivate 2",
		"chown -R ceph:ceph /var/lib/ceph",
		)

		# Start OSDs (again, argon2id eats memory, so this happens before starting the mgr)
		ceph.succeed(
		"systemctl start ceph-osd-0.service",
		"systemctl start ceph-osd-1.service",
		"systemctl start ceph-osd-2.service",
		)
		ceph.wait_until_succeeds("ceph -s \| grep 'quorum 0'")
		ceph.wait_until_succeeds("ceph osd stat \| grep -e '3 osds: 3 up[^,]*, 3 in'")

		# Start the ceph-mgr daemon, after copying in the keyring
		ceph.succeed(
		"mkdir -p /var/lib/ceph/mgr/ceph-ceph/",
		"ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow ' mds 'allow '",
		"chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/",
		"systemctl start ceph-mgr-ceph.service",
		)
		ceph.wait_for_unit("ceph-mgr-ceph")
		ceph.wait_until_succeeds("ceph -s \| grep 'quorum 0'")
		ceph.wait_until_succeeds("ceph -s \| grep 'mgr: ceph(active,'")
		ceph.wait_until_succeeds("ceph osd stat \| grep -e '3 osds: 3 up[^,]*, 3 in'")
		ceph.wait_until_succeeds("ceph -s \| grep 'HEALTH_OK'")

		# test the actual storage
		ceph.succeed(
		"ceph osd pool create single-node-test 32 32",
		"ceph osd pool ls \| grep 'single-node-test'",

		# We need to enable an application on the pool, otherwise it will
		# stay unhealthy in state POOL_APP_NOT_ENABLED.
		# Creating a CephFS would do this automatically, but we haven't done that here.
		# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
		# We use the custom application name "nixos-test" for this.
		"ceph osd pool application enable single-node-test nixos-test",

		"ceph osd pool rename single-node-test single-node-other-test",
		"ceph osd pool ls \| grep 'single-node-other-test'",
		)
		ceph.wait_until_succeeds("ceph -s \| grep '2 pools, 33 pgs'")
		ceph.wait_until_succeeds("ceph -s \| grep 'HEALTH_OK'")
		ceph.wait_until_succeeds("ceph -s \| grep '33 active+clean'")
		ceph.fail(
		# the old pool should be gone
		"ceph osd pool ls \| grep 'multi-node-test'",
		# deleting the pool should fail without setting mon_allow_pool_delete
		"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
		)

		# rebooting gets rid of any potential tmpfs mounts or device-mapper devices
		ceph.shutdown()
		ceph.start()
		ceph.wait_for_unit("default.target")

		# Start it up (again OSDs first due to memory constraints of cryptsetup and mgr)
		ceph.systemctl("start ceph-mon-0.service")
		ceph.wait_for_unit("ceph-mon-0")
		ceph.systemctl("start ceph-osd-0.service")
		ceph.wait_for_unit("ceph-osd-0")
		ceph.systemctl("start ceph-osd-1.service")
		ceph.wait_for_unit("ceph-osd-1")
		ceph.systemctl("start ceph-osd-2.service")
		ceph.wait_for_unit("ceph-osd-2")
		ceph.systemctl("start ceph-mgr-ceph.service")
		ceph.wait_for_unit("ceph-mgr-ceph")

		# Ensure the cluster comes back up again
		ceph.succeed("ceph -s \| grep 'mon: 1 daemons'")
		ceph.wait_until_succeeds("ceph -s \| grep 'quorum 0'")
		ceph.wait_until_succeeds("ceph osd stat \| grep -E '3 osds: 3 up[^,]*, 3 in'")
		ceph.wait_until_succeeds("ceph -s \| grep 'mgr: ceph(active,'")
		ceph.wait_until_succeeds("ceph -s \| grep 'HEALTH_OK'")
		'';
		}
		)

pkgs/tools/filesystems/ceph/default.nix

+7 −1

Original line number	Diff line number	Diff line
		@@ -323,6 +323,11 @@ in rec {
		name = "ceph-reef-ceph-volume-fix-set_dmcrypt_no_workqueue.patch";
		hash = "sha256-r+7hcCz2WF/rJfgKwTatKY9unJlE8Uw3fmOyaY5jVH0=";
		})
		(fetchpatch {
		url = "https://github.com/ceph/ceph/commit/607eb34b2c278566c386efcbf3018629cf08ccfd.patch";
		name = "ceph-volume-fix-set_dmcrypt_no_workqueue-regex.patch";
		hash = "sha256-q28Q7OIyFoMyMBCPXGA+AdNqp+9/6J/XwD4ODjx+JXY=";
		})
		];

		postPatch = ''
		@@ -487,7 +492,8 @@ in rec {
		inherit (nixosTests)
		ceph-multi-node
		ceph-single-node
		ceph-single-node-bluestore;
		ceph-single-node-bluestore
		ceph-single-node-bluestore-dmcrypt;
		};
		};
		};