nixos/vector: Add Caddy/DNSTAP+ClickHouse test (#417234) (68b71e75) · Commits · nix / nixpkgs

nixos/tests/all-tests.nix

+1 −1

Original line number	Diff line number	Diff line
		@@ -1464,7 +1464,7 @@ in
		vault-postgresql = runTest ./vault-postgresql.nix;
		vaultwarden = discoverTests (import ./vaultwarden.nix);
		vdirsyncer = runTest ./vdirsyncer.nix;
		vector = handleTest ./vector { };
		vector = import ./vector { inherit runTest; };
		velocity = runTest ./velocity.nix;
		vengi-tools = runTest ./vengi-tools.nix;
		victoriametrics = handleTest ./victoriametrics { };

nixos/tests/vector/api.nix

+32 −34

Original line number	Diff line number	Diff line
		import ../make-test-python.nix (
		{ lib, pkgs, ... }:

		{
		@@ -42,4 +41,3 @@ import ../make-test-python.nix (
		machineapi.wait_until_succeeds("curl -sSf http://localhost:8686/health")
		'';
		}
		)

nixos/tests/vector/caddy-clickhouse.nix

0 → 100644

+206 −0

Original line number	Diff line number	Diff line
		{ lib, pkgs, ... }:

		{
		name = "vector-caddy-clickhouse";
		meta.maintainers = [ pkgs.lib.maintainers.happysalada ];

		nodes = {
		caddy =
		{ config, pkgs, ... }:
		{
		networking.firewall.allowedTCPPorts = [ 80 ];

		services.caddy = {
		enable = true;
		virtualHosts = {
		"http://caddy" = {
		extraConfig = ''
		encode gzip

		file_server
		root /srv
		'';
		logFormat = "
		output file ${config.services.caddy.logDir}/access-caddy.log {
		mode 0640
		}
		";
		};
		};
		};

		systemd.services.vector.serviceConfig = {
		SupplementaryGroups = [ "caddy" ];
		};

		services.vector = {
		enable = true;

		settings = {
		sources = {
		caddy-log = {
		type = "file";
		include = [ "/var/log/caddy/*.log" ];
		};
		};

		transforms = {
		caddy_logs_timestamp = {
		type = "remap";
		inputs = [ "caddy-log" ];
		source = ''
		.tmp_timestamp, err = parse_json!(.message).ts * 1000000

		if err != null {
		log("Unable to parse ts value: " + err, level: "error")
		} else {
		.timestamp = from_unix_timestamp!(to_int!(.tmp_timestamp), unit: "microseconds")
		}

		del(.tmp_timestamp)
		'';
		};
		};

		sinks = {
		vector_sink = {
		type = "vector";
		inputs = [ "caddy_logs_timestamp" ];
		address = "clickhouse:6000";
		};
		};
		};
		};
		};

		client =
		{ config, pkgs, ... }:
		{
		environment.systemPackages = [ pkgs.curl ];
		};

		clickhouse =
		{ config, pkgs, ... }:
		{
		virtualisation.memorySize = 4096;

		networking.firewall.allowedTCPPorts = [ 6000 ];

		services.vector = {
		enable = true;

		settings = {
		sources = {
		vector_source = {
		type = "vector";
		address = "[::]:6000";
		};
		};

		sinks = {
		clickhouse = {
		type = "clickhouse";
		inputs = [
		"vector_source"
		];
		endpoint = "http://localhost:8123";
		database = "caddy";
		table = "access_logs";
		date_time_best_effort = true;
		skip_unknown_fields = true;
		};
		};
		};

		};

		services.clickhouse = {
		enable = true;
		};
		};
		};

		testScript =
		let
		# work around quote/substitution complexity by Nix, Perl, bash and SQL.
		databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS caddy";

		tableDDL = pkgs.writeText "table.sql" ''
		CREATE TABLE IF NOT EXISTS caddy.access_logs (
		timestamp DateTime64(6),
		host LowCardinality(String),
		message String,
		)
		ENGINE = MergeTree()
		ORDER BY timestamp
		PARTITION BY toYYYYMM(timestamp)
		'';

		tableViewBase = pkgs.writeText "table-view-base.sql" ''
		CREATE TABLE IF NOT EXISTS caddy.access_logs_view_base (
		timestamp DateTime64(6),
		host LowCardinality(String),
		request JSON,
		status UInt16,
		)
		ENGINE = MergeTree()
		ORDER BY timestamp
		PARTITION BY toYYYYMM(timestamp)
		'';

		tableView = pkgs.writeText "table-view.sql" ''
		CREATE MATERIALIZED VIEW IF NOT EXISTS caddy.access_logs_view TO caddy.access_logs_view_base
		AS SELECT
		timestamp,
		host,
		simpleJSONExtractRaw(message, 'request') AS request,
		simpleJSONExtractRaw(message, 'status') AS status
		FROM caddy.access_logs;
		'';

		selectQuery = pkgs.writeText "select.sql" ''
		SELECT
		timestamp,
		request.host,
		request.remote_ip,
		request.proto,
		request.method,
		request.uri,
		status
		FROM caddy.access_logs_view_base
		WHERE request.uri LIKE '%test-uri%'
		FORMAT Pretty
		'';
		in
		''
		clickhouse.wait_for_unit("clickhouse")
		clickhouse.wait_for_unit("vector")
		clickhouse.wait_for_open_port(6000)
		clickhouse.wait_for_open_port(8123)

		clickhouse.succeed(
		"cat ${databaseDDL} \| clickhouse-client",
		"cat ${tableDDL} \| clickhouse-client",
		"cat ${tableViewBase} \| clickhouse-client",
		"cat ${tableView} \| clickhouse-client",
		)

		caddy.wait_for_unit("caddy")
		caddy.wait_for_open_port(80)
		caddy.wait_for_unit("vector")
		caddy.wait_until_succeeds(
		"journalctl -o cat -u vector.service \| grep 'Vector has started'"
		)

		client.systemctl("start network-online.target")
		client.wait_until_succeeds("curl http://caddy/test-uri")

		caddy.wait_until_succeeds(
		"journalctl -o cat -u vector.service \| grep 'Found new file to watch. file=/var/log/caddy/access-caddy.log'"
		)

		clickhouse.wait_until_succeeds(
		"cat ${selectQuery} \| clickhouse-client \| grep test-uri"
		)
		'';
		}

nixos/tests/vector/default.nix

+8 −11

Original line number	Diff line number	Diff line
		{
		system ? builtins.currentSystem,
		config ? { },
		pkgs ? import ../../.. { inherit system config; },
		}:
		{ runTest }:

		{
		file-sink = import ./file-sink.nix { inherit system pkgs; };
		api = import ./api.nix { inherit system pkgs; };
		dnstap = import ./dnstap.nix { inherit system pkgs; };
		journald-clickhouse = import ./journald-clickhouse.nix { inherit system pkgs; };
		nginx-clickhouse = import ./nginx-clickhouse.nix { inherit system pkgs; };
		syslog-quickwit = import ./syslog-quickwit.nix { inherit system pkgs; };
		file-sink = runTest ./file-sink.nix;
		api = runTest ./api.nix;
		caddy-clickhouse = runTest ./caddy-clickhouse.nix;
		dnstap = runTest ./dnstap.nix;
		journald-clickhouse = runTest ./journald-clickhouse.nix;
		nginx-clickhouse = runTest ./nginx-clickhouse.nix;
		syslog-quickwit = runTest ./syslog-quickwit.nix;
		}

nixos/tests/vector/dnstap.nix

+191 −84

Original line number	Diff line number	Diff line
		import ../make-test-python.nix (
		{ lib, pkgs, ... }:

		let
		@@ -9,6 +8,40 @@ import ../make-test-python.nix (
		meta.maintainers = [ pkgs.lib.maintainers.happysalada ];

		nodes = {
		clickhouse =
		{ config, pkgs, ... }:
		{
		networking.firewall.allowedTCPPorts = [ 6000 ];

		services.vector = {
		enable = true;

		settings = {
		sources = {
		vector_dnstap_source = {
		type = "vector";
		address = "[::]:6000";
		};
		};

		sinks = {
		clickhouse = {
		type = "clickhouse";
		inputs = [
		"vector_dnstap_source"
		];
		endpoint = "http://localhost:8123";
		database = "dnstap";
		table = "records";
		date_time_best_effort = true;
		};
		};
		};
		};

		services.clickhouse.enable = true;
		};

		unbound =
		{ config, pkgs, ... }:
		{
		@@ -38,6 +71,12 @@ import ../make-test-python.nix (
		codec = "json";
		};
		};

		vector_dnstap_sink = {
		type = "vector";
		inputs = [ "dnstap" ];
		address = "clickhouse:6000";
		};
		};
		};
		};
		@@ -100,7 +139,72 @@ import ../make-test-python.nix (
		};
		};

		testScript = ''
		testScript =
		let
		# work around quote/substitution complexity by Nix, Perl, bash and SQL.
		databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS dnstap";

		tableDDL = pkgs.writeText "table.sql" ''
		CREATE TABLE IF NOT EXISTS dnstap.records (
		timestamp DateTime64(6),
		dataType LowCardinality(String),
		dataTypeId UInt8,
		messageType LowCardinality(String),
		messageTypeId UInt8,
		requestData Nullable(JSON),
		responseData Nullable(JSON),
		responsePort UInt16,
		serverId LowCardinality(String),
		serverVersion LowCardinality(String),
		socketFamily LowCardinality(String),
		socketProtocol LowCardinality(String),
		sourceAddress String,
		sourcePort UInt16,
		)
		ENGINE = MergeTree()
		ORDER BY (serverId, timestamp)
		PARTITION BY toYYYYMM(timestamp)
		'';

		tableView = pkgs.writeText "view.sql" ''
		CREATE MATERIALIZED VIEW dnstap.domains_view (
		timestamp DateTime64(6),
		serverId LowCardinality(String),
		domain String,
		record_type LowCardinality(String)
		)
		ENGINE = MergeTree()
		PARTITION BY toYYYYMM(timestamp)
		ORDER BY (serverId, timestamp)
		POPULATE AS
		SELECT
		timestamp,
		serverId,
		JSONExtractString(requestData.question[1]::String, 'domainName') as domain,
		JSONExtractString(requestData.question[1]::String, 'questionType') as record_type
		FROM dnstap.records
		WHERE messageTypeId = 5 # ClientQuery
		'';

		selectQuery = pkgs.writeText "select.sql" ''
		SELECT
		domain,
		count(domain)
		FROM dnstap.domains_view
		GROUP BY domain
		'';
		in
		''
		clickhouse.wait_for_unit("clickhouse")
		clickhouse.wait_for_open_port(6000)
		clickhouse.wait_for_open_port(8123)

		clickhouse.succeed(
		"cat ${databaseDDL} \| clickhouse-client",
		"cat ${tableDDL} \| clickhouse-client",
		"cat ${tableView} \| clickhouse-client",
		)

		unbound.wait_for_unit("unbound")
		unbound.wait_for_unit("vector")

		@@ -128,6 +232,9 @@ import ../make-test-python.nix (
		unbound.wait_until_succeeds(
		"grep ClientResponse /var/lib/vector/logs.log \| grep '\"domainName\":\"test.local.\"' \| grep '\"rData\":\"192.168.123.5\"'"
		)

		clickhouse.log(clickhouse.wait_until_succeeds(
		"cat ${selectQuery} \| clickhouse-client \| grep 'test.local.'"
		))
		'';
		}
		)

Admin message