Merge pull request #144239 from illustris/hadoop (1403ce52) · Commits · nix / nixpkgs

nixos/doc/manual/from_md/release-notes/rl-2111.section.xml

+23 −0

Original line number	Diff line number	Diff line
		@@ -50,6 +50,29 @@
		guide</link> is available.
		</para>
		</listitem>
		<listitem>
		<para>
		Improvements have been made to the Hadoop module and package:
		</para>
		<itemizedlist spacing="compact">
		<listitem>
		<para>
		HDFS and YARN now support production-ready highly
		available deployments with automatic failover.
		</para>
		</listitem>
		<listitem>
		<para>
		Hadoop now defaults to Hadoop 3, updated from 2.
		</para>
		</listitem>
		<listitem>
		<para>
		JournalNode, ZKFS and HTTPFS services have been added.
		</para>
		</listitem>
		</itemizedlist>
		</listitem>
		<listitem>
		<para>
		Activation scripts can now opt int to be run when running

nixos/doc/manual/release-notes/rl-2111.section.md

+5 −0

Original line number	Diff line number	Diff line
		@@ -18,6 +18,11 @@ In addition to numerous new and upgraded packages, this release has the followin

		- spark now defaults to spark 3, updated from 2. A [migration guide](https://spark.apache.org/docs/latest/core-migration-guide.html#upgrading-from-core-24-to-30) is available.

		- Improvements have been made to the Hadoop module and package:
		- HDFS and YARN now support production-ready highly available deployments with automatic failover.
		- Hadoop now defaults to Hadoop 3, updated from 2.
		- JournalNode, ZKFS and HTTPFS services have been added.

		- Activation scripts can now opt int to be run when running `nixos-rebuild dry-activate` and detect the dry activation by reading `$NIXOS_ACTION`.
		This allows activation scripts to output what they would change if the activation was really run.
		The users/modules activation script supports this and outputs some of is actions.

nixos/modules/services/cluster/hadoop/conf.nix

+1 −0

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@ pkgs.runCommand "hadoop-conf" {} ''
		cp ${siteXml "hdfs-site.xml" cfg.hdfsSite}/* $out/
		cp ${siteXml "mapred-site.xml" cfg.mapredSite}/* $out/
		cp ${siteXml "yarn-site.xml" cfg.yarnSite}/* $out/
		cp ${siteXml "httpfs-site.xml" cfg.httpfsSite}/* $out/
		cp ${cfgFile "container-executor.cfg" cfg.containerExecutorCfg}/* $out/
		cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/
		cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/

nixos/modules/services/cluster/hadoop/default.nix

+36 −6

Original line number	Diff line number	Diff line
		@@ -15,7 +15,10 @@ with lib;
		"fs.defaultFS" = "hdfs://localhost";
		}
		'';
		description = "Hadoop core-site.xml definition";
		description = ''
		Hadoop core-site.xml definition
		<link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml"/>
		'';
		};

		hdfsSite = mkOption {
		@@ -28,7 +31,10 @@ with lib;
		"dfs.nameservices" = "namenode1";
		}
		'';
		description = "Hadoop hdfs-site.xml definition";
		description = ''
		Hadoop hdfs-site.xml definition
		<link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
		'';
		};

		mapredSite = mkOption {
		@@ -44,7 +50,10 @@ with lib;
		"mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
		}
		'';
		description = "Hadoop mapred-site.xml definition";
		description = ''
		Hadoop mapred-site.xml definition
		<link xlink:href="https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
		'';
		};

		yarnSite = mkOption {
		@@ -67,7 +76,24 @@ with lib;
		"yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
		}
		'';
		description = "Hadoop yarn-site.xml definition";
		description = ''
		Hadoop yarn-site.xml definition
		<link xlink:href="https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
		'';
		};

		httpfsSite = mkOption {
		default = { };
		type = types.attrsOf types.anything;
		example = literalExpression ''
		{
		"hadoop.http.max.threads" = 500;
		}
		'';
		description = ''
		Hadoop httpfs-site.xml definition
		<link xlink:href="https://hadoop.apache.org/docs/current/hadoop-hdfs-httpfs/httpfs-default.html"/>
		'';
		};

		log4jProperties = mkOption {
		@@ -92,7 +118,10 @@ with lib;
		"feature.terminal.enabled" = 0;
		}
		'';
		description = "Yarn container-executor.cfg definition";
		description = ''
		Yarn container-executor.cfg definition
		<link xlink:href="https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/>
		'';
		};

		extraConfDirs = mkOption {
		@@ -118,7 +147,8 @@ with lib;

		config = mkMerge [
		(mkIf (builtins.hasAttr "yarn" config.users.users \|\|
		builtins.hasAttr "hdfs" config.users.users) {
		builtins.hasAttr "hdfs" config.users.users \|\|
		builtins.hasAttr "httpfs" config.users.users) {
		users.groups.hadoop = {
		gid = config.ids.gids.hadoop;
		};

nixos/modules/services/cluster/hadoop/hdfs.nix

+105 −12

Original line number	Diff line number	Diff line
		@@ -17,11 +17,14 @@ in
		{
		options.services.hadoop.hdfs = {
		namenode = {
		enabled = mkOption {
		enable = mkEnableOption "Whether to run the HDFS NameNode";
		formatOnInit = mkOption {
		type = types.bool;
		default = false;
		description = ''
		Whether to run the HDFS NameNode
		Format HDFS namenode on first start. This is useful for quickly spinning up ephemeral HDFS clusters with a single namenode.
		For HA clusters, initialization involves multiple steps across multiple nodes. Follow [this guide](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html)
		to initialize an HA cluster manually.
		'';
		};
		inherit restartIfChanged;
		@@ -34,34 +37,61 @@ in
		};
		};
		datanode = {
		enabled = mkOption {
		enable = mkEnableOption "Whether to run the HDFS DataNode";
		inherit restartIfChanged;
		openFirewall = mkOption {
		type = types.bool;
		default = false;
		default = true;
		description = ''
		Whether to run the HDFS DataNode
		Open firewall ports for datanode
		'';
		};
		};
		journalnode = {
		enable = mkEnableOption "Whether to run the HDFS JournalNode";
		inherit restartIfChanged;
		openFirewall = mkOption {
		type = types.bool;
		default = true;
		description = ''
		Open firewall ports for datanode
		Open firewall ports for journalnode
		'';
		};
		};
		zkfc = {
		enable = mkEnableOption "Whether to run the HDFS ZooKeeper failover controller";
		inherit restartIfChanged;
		};
		httpfs = {
		enable = mkEnableOption "Whether to run the HDFS HTTPfs server";
		tempPath = mkOption {
		type = types.path;
		default = "/tmp/hadoop/httpfs";
		description = ''
		HTTPFS_TEMP path used by HTTPFS
		'';
		};
		inherit restartIfChanged;
		openFirewall = mkOption {
		type = types.bool;
		default = true;
		description = ''
		Open firewall ports for HTTPFS
		'';
		};
		};
		};

		config = mkMerge [
		(mkIf cfg.hdfs.namenode.enabled {
		(mkIf cfg.hdfs.namenode.enable {
		systemd.services.hdfs-namenode = {
		description = "Hadoop HDFS NameNode";
		wantedBy = [ "multi-user.target" ];
		inherit (cfg.hdfs.namenode) restartIfChanged;

		preStart = ''
		preStart = (mkIf cfg.hdfs.namenode.formatOnInit ''
		${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive \|\| true
		'';
		'');

		serviceConfig = {
		User = "hdfs";
		@@ -74,9 +104,10 @@ in
		networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.namenode.openFirewall [
		9870 # namenode.http-address
		8020 # namenode.rpc-address
		8022 # namenode. servicerpc-address
		]);
		})
		(mkIf cfg.hdfs.datanode.enabled {
		(mkIf cfg.hdfs.datanode.enable {
		systemd.services.hdfs-datanode = {
		description = "Hadoop HDFS DataNode";
		wantedBy = [ "multi-user.target" ];
		@@ -96,8 +127,64 @@ in
		9867 # datanode.ipc.address
		]);
		})
		(mkIf cfg.hdfs.journalnode.enable {
		systemd.services.hdfs-journalnode = {
		description = "Hadoop HDFS JournalNode";
		wantedBy = [ "multi-user.target" ];
		inherit (cfg.hdfs.journalnode) restartIfChanged;

		serviceConfig = {
		User = "hdfs";
		SyslogIdentifier = "hdfs-journalnode";
		ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} journalnode";
		Restart = "always";
		};
		};

		networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.journalnode.openFirewall [
		8480 # dfs.journalnode.http-address
		8485 # dfs.journalnode.rpc-address
		]);
		})
		(mkIf cfg.hdfs.zkfc.enable {
		systemd.services.hdfs-zkfc = {
		description = "Hadoop HDFS ZooKeeper failover controller";
		wantedBy = [ "multi-user.target" ];
		inherit (cfg.hdfs.zkfc) restartIfChanged;

		serviceConfig = {
		User = "hdfs";
		SyslogIdentifier = "hdfs-zkfc";
		ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} zkfc";
		Restart = "always";
		};
		};
		})
		(mkIf cfg.hdfs.httpfs.enable {
		systemd.services.hdfs-httpfs = {
		description = "Hadoop httpfs";
		wantedBy = [ "multi-user.target" ];
		inherit (cfg.hdfs.httpfs) restartIfChanged;

		environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;

		preStart = ''
		mkdir -p $HTTPFS_TEMP
		'';

		serviceConfig = {
		User = "httpfs";
		SyslogIdentifier = "hdfs-httpfs";
		ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} httpfs";
		Restart = "always";
		};
		};
		networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.httpfs.openFirewall [
		14000 # httpfs.http.port
		]);
		})
		(mkIf (
		cfg.hdfs.namenode.enabled \|\| cfg.hdfs.datanode.enabled
		cfg.hdfs.namenode.enable \|\| cfg.hdfs.datanode.enable \|\| cfg.hdfs.journalnode.enable \|\| cfg.hdfs.zkfc.enable
		) {
		users.users.hdfs = {
		description = "Hadoop HDFS user";
		@@ -105,6 +192,12 @@ in
		uid = config.ids.uids.hdfs;
		};
		})

		(mkIf cfg.hdfs.httpfs.enable {
		users.users.httpfs = {
		description = "Hadoop HTTPFS user";
		group = "hadoop";
		isSystemUser = true;
		};
		})
		];
		}

Admin message