Merge pull request #316727 from jpds/nixos-tests-prometheus (3c033186) · Commits · nix / nixpkgs

maintainers/maintainer-list.nix

+5 −0

Original line number	Diff line number	Diff line
		@@ -9848,6 +9848,11 @@
		githubId = 1918771;
		name = "Joe Doyle";
		};
		jpds = {
		github = "jpds";
		githubId = 29158971;
		name = "Jonathan Davies";
		};
		jpentland = {
		email = "joe.pentland@gmail.com";
		github = "jpentland";

nixos/modules/module-list.nix

+1 −0

Original line number	Diff line number	Diff line
		@@ -880,6 +880,7 @@
		./services/monitoring/osquery.nix
		./services/monitoring/parsedmarc.nix
		./services/monitoring/prometheus/alertmanager-irc-relay.nix
		./services/monitoring/prometheus/alertmanager-webhook-logger.nix
		./services/monitoring/prometheus/alertmanager.nix
		./services/monitoring/prometheus/default.nix
		./services/monitoring/prometheus/exporters.nix

nixos/modules/services/monitoring/prometheus/alertmanager-webhook-logger.nix

0 → 100644

+70 −0

Original line number	Diff line number	Diff line
		{ config, lib, pkgs, ... }:

		with lib;

		let
		cfg = config.services.prometheus.alertmanagerWebhookLogger;
		in
		{
		options.services.prometheus.alertmanagerWebhookLogger = {
		enable = mkEnableOption "Alertmanager Webhook Logger";

		package = mkPackageOption pkgs "alertmanager-webhook-logger" { };

		extraFlags = mkOption {
		type = types.listOf types.str;
		default = [];
		description = "Extra command line options to pass to alertmanager-webhook-logger.";
		};
		};

		config = mkIf cfg.enable {
		systemd.services.alertmanager-webhook-logger = {
		description = "Alertmanager Webhook Logger";

		wantedBy = [ "multi-user.target" ];
		after = [ "network-online.target" ];
		wants = [ "network-online.target" ];

		serviceConfig = {
		ExecStart = ''
		${cfg.package}/bin/alertmanager-webhook-logger \
		${escapeShellArgs cfg.extraFlags}
		'';

		DynamicUser = true;
		NoNewPrivileges = true;

		ProtectProc = "invisible";
		ProtectSystem = "strict";
		ProtectHome = "tmpfs";

		PrivateTmp = true;
		PrivateDevices = true;
		PrivateIPC = true;

		ProtectHostname = true;
		ProtectClock = true;
		ProtectKernelTunables = true;
		ProtectKernelModules = true;
		ProtectKernelLogs = true;
		ProtectControlGroups = true;

		RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
		RestrictRealtime = true;
		RestrictSUIDSGID = true;

		SystemCallFilter = [
		"@system-service"
		"~@cpu-emulation"
		"~@privileged"
		"~@reboot"
		"~@setuid"
		"~@swap"
		];
		};
		};
		};

		meta.maintainers = [ maintainers.jpds ];
		}

nixos/tests/all-tests.nix

+2 −1

Original line number	Diff line number	Diff line
		@@ -775,7 +775,7 @@ in {
		printing-service = handleTest ./printing.nix { socket = false; };
		private-gpt = handleTest ./private-gpt.nix {};
		privoxy = handleTest ./privoxy.nix {};
		prometheus = handleTest ./prometheus.nix {};
		prometheus = handleTest ./prometheus {};
		prometheus-exporters = handleTest ./prometheus-exporters.nix {};
		prosody = handleTest ./xmpp/prosody.nix {};
		prosody-mysql = handleTest ./xmpp/prosody-mysql.nix {};
		@@ -968,6 +968,7 @@ in {
		teleport = handleTest ./teleport.nix {};
		thelounge = handleTest ./thelounge.nix {};
		terminal-emulators = handleTest ./terminal-emulators.nix {};
		thanos = handleTest ./thanos.nix {};
		tiddlywiki = handleTest ./tiddlywiki.nix {};
		tigervnc = handleTest ./tigervnc.nix {};
		timescaledb = handleTest ./timescaledb.nix {};

nixos/tests/prometheus/alertmanager.nix

0 → 100644

+148 −0

Original line number	Diff line number	Diff line
		import ../make-test-python.nix ({ lib, pkgs, ... }:

		{
		name = "prometheus-alertmanager";

		nodes = {
		prometheus = { config, pkgs, ... }: {
		environment.systemPackages = [ pkgs.jq ];

		networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];

		services.prometheus = {
		enable = true;
		globalConfig.scrape_interval = "2s";

		alertmanagers = [
		{
		scheme = "http";
		static_configs = [
		{
		targets = [
		"alertmanager:${toString config.services.prometheus.alertmanager.port}"
		];
		}
		];
		}
		];

		rules = [
		''
		groups:
		- name: test
		rules:
		- alert: InstanceDown
		expr: up == 0
		for: 5s
		labels:
		severity: page
		annotations:
		summary: "Instance {{ $labels.instance }} down"
		''
		];

		scrapeConfigs = [
		{
		job_name = "alertmanager";
		static_configs = [
		{
		targets = [
		"alertmanager:${toString config.services.prometheus.alertmanager.port}"
		];
		}
		];
		}
		{
		job_name = "node";
		static_configs = [
		{
		targets = [
		"node:${toString config.services.prometheus.exporters.node.port}"
		];
		}
		];
		}
		];
		};
		};

		alertmanager = { config, pkgs, ... }: {
		services.prometheus.alertmanager = {
		enable = true;
		openFirewall = true;

		configuration = {
		global = {
		resolve_timeout = "1m";
		};

		route = {
		# Root route node
		receiver = "test";
		group_by = ["..."];
		continue = false;
		group_wait = "1s";
		group_interval = "15s";
		repeat_interval = "24h";
		};

		receivers = [
		{
		name = "test";
		webhook_configs = [
		{
		url = "http://logger:6725";
		send_resolved = true;
		max_alerts = 0;
		}
		];
		}
		];
		};
		};
		};

		logger = { config, pkgs, ... }: {
		networking.firewall.allowedTCPPorts = [ 6725 ];

		services.prometheus.alertmanagerWebhookLogger.enable = true;
		};
		};

		testScript = ''
		alertmanager.wait_for_unit("alertmanager")
		alertmanager.wait_for_open_port(9093)
		alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready")
		#alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service \| grep 'version=${pkgs.prometheus-alertmanager.version}'")

		logger.wait_for_unit("alertmanager-webhook-logger")
		logger.wait_for_open_port(6725)

		prometheus.wait_for_unit("prometheus")
		prometheus.wait_for_open_port(9090)

		prometheus.wait_until_succeeds(
		"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' \| "
		+ "jq '.data.result[0].value[1]' \| grep '\"1\"'"
		)

		prometheus.wait_until_succeeds(
		"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' \| "
		+ "jq '.data.result[0].metric.version' \| grep '\"${pkgs.prometheus-alertmanager.version}\"'"
		)

		prometheus.wait_until_succeeds(
		"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' \| "
		+ "jq '.data.result[0].value[1]' \| grep '\"1\"'"
		)

		prometheus.wait_until_succeeds(
		"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' \| "
		+ "jq '.data.result[0].value[1]' \| grep -v '\"0\"'"
		)

		logger.wait_until_succeeds(
		"journalctl -o cat -u alertmanager-webhook-logger.service \| grep '\"alertname\":\"InstanceDown\"'"
		)
		'';
		})

Admin message