nixos/ncps: Add all options available and use config file (#481209) (2dab65fe) · Commits · nix / nixpkgs

nixos/modules/services/networking/ncps.nix

+225 −88

Original line number	Diff line number	Diff line
		@@ -18,89 +18,110 @@ let
		];

		ncpsWrapper = pkgs.writeShellScript "ncps-wrapper" ''
		${lib.optionalString (cfg.cache.secretKeyPath != null) ''
		export CACHE_SECRET_KEY_PATH="$CREDENTIALS_DIRECTORY/secretKey"
		''}

		${lib.optionalString (cfg.cache.storage.s3 != null) ''
		export CACHE_STORAGE_S3_ACCESS_KEY_ID="$(cat "$CREDENTIALS_DIRECTORY/s3AccessKeyId")"
		export CACHE_STORAGE_S3_SECRET_ACCESS_KEY="$(cat "$CREDENTIALS_DIRECTORY/s3SecretAccessKey")"
		''}

		${lib.optionalString (cfg.cache.redis != null && cfg.cache.redis.passwordFile != null) ''
		export CACHE_REDIS_PASSWORD="$(cat "$CREDENTIALS_DIRECTORY/redisPassword")"
		''}
		${lib.optionalString (cfg.cache.redis != null) (
		if cfg.cache.redis.passwordFile != null then
		''export CACHE_REDIS_PASSWORD="$(cat "$CREDENTIALS_DIRECTORY/redisPassword")"''
		else if cfg.cache.redis.password != null then
		''export CACHE_REDIS_PASSWORD="${cfg.cache.redis.password}"''
		else
		""
		)}

		${lib.optionalString (cfg.cache.databaseURLFile != null) ''
		export CACHE_DATABASE_URL="$(cat "$CREDENTIALS_DIRECTORY/databaseURL")"
		''}

		exec ${lib.getExe cfg.package} "$@"
		'';

		globalFlags = lib.concatStringsSep " " (
		[ "--log-level='${cfg.logLevel}'" ]
		++ (lib.optionals cfg.openTelemetry.enable (
		[
		"--otel-enabled"
		]
		++ (lib.optional (
		cfg.openTelemetry.grpcURL != null
		) "--otel-grpc-url='${cfg.openTelemetry.grpcURL}'")
		))
		++ (lib.optional cfg.prometheus.enable "--prometheus-enabled")
		++ (lib.optional (!cfg.analytics.reporting.enable) "--analytics-reporting-enabled=false")
		);
		exec ${lib.getExe cfg.package} --config "${configFile}" "$@"
		'';

		serveFlags = lib.concatStringsSep " " (
		[
		"--cache-hostname='${cfg.cache.hostName}'"
		"--cache-lock-backend='${cfg.cache.lock.backend}'"
		"--cache-temp-path='${cfg.cache.tempPath}'"
		"--server-addr='${cfg.server.addr}'"
		]
		++ (lib.optional (cfg.cache.databaseURL != null) "--cache-database-url='${cfg.cache.databaseURL}'")
		++ (lib.optionals (cfg.cache.redis != null) (
		[
		"--cache-redis-addrs='${builtins.concatStringsSep "," cfg.cache.redis.addresses}'"
		"--cache-redis-db='${builtins.toString cfg.cache.redis.database}'"
		"--cache-redis-pool-size='${builtins.toString cfg.cache.redis.poolSize}'"
		]
		++ (lib.optional (
		cfg.cache.redis.username != null
		) "--cache-redis-username='${cfg.cache.redis.username}'")
		++ (lib.optional (
		cfg.cache.redis.password != null
		) "--cache-redis-password='${cfg.cache.redis.password}'")
		++ (lib.optional cfg.cache.redis.useTLS "--cache-redis-use-tls")
		))
		++ (lib.optional (
		cfg.cache.storage.s3 == null
		) "--cache-storage-local='${cfg.cache.storage.local}'")
		++ (lib.optionals (cfg.cache.storage.s3 != null) (
		[
		"--cache-storage-s3-bucket='${cfg.cache.storage.s3.bucket}'"
		"--cache-storage-s3-endpoint='${cfg.cache.storage.s3.endpoint}'"
		]
		++ (lib.optional cfg.cache.storage.s3.forcePathStyle "--cache-storage-s3-force-path-style")
		++ (lib.optional (
		cfg.cache.storage.s3.region != null
		) "--cache-storage-s3-region='${cfg.cache.storage.s3.region}'")
		))
		++ (lib.optional cfg.cache.allowDeleteVerb "--cache-allow-delete-verb")
		++ (lib.optional cfg.cache.allowPutVerb "--cache-allow-put-verb")
		++ (lib.optional (cfg.cache.maxSize != null) "--cache-max-size='${cfg.cache.maxSize}'")
		++ (lib.optionals (cfg.cache.lru.schedule != null) [
		"--cache-lru-schedule='${cfg.cache.lru.schedule}'"
		"--cache-lru-schedule-timezone='${cfg.cache.lru.scheduleTimeZone}'"
		])
		++ (lib.optional (cfg.cache.secretKeyPath != null) "--cache-secret-key-path='%d/secretKey'")
		++ (lib.optional (!cfg.cache.signNarinfo) "--cache-sign-narinfo='false'")
		++ (lib.optional (
		cfg.cache.upstream.dialerTimeout != null
		) "--cache-upstream-dialer-timeout='${cfg.cache.upstream.dialerTimeout}'")
		++ (lib.optional (
		cfg.cache.upstream.responseHeaderTimeout != null
		) "--cache-upstream-response-header-timeout='${cfg.cache.upstream.responseHeaderTimeout}'")
		++ (lib.forEach cfg.cache.upstream.publicKeys (pk: "--cache-upstream-public-key='${pk}'"))
		++ (lib.forEach cfg.cache.upstream.urls (url: "--cache-upstream-url='${url}'"))
		++ (lib.optional (cfg.netrcFile != null) "--netrc-file='${cfg.netrcFile}'")
		settings = {
		log.level = cfg.logLevel;
		opentelemetry = lib.optionalAttrs cfg.openTelemetry.enable {
		enabled = true;
		grpc-url = cfg.openTelemetry.grpcURL;
		};
		prometheus = lib.optionalAttrs cfg.prometheus.enable {
		enabled = true;
		};
		analytics.reporting = {
		enabled = cfg.analytics.reporting.enable;
		samples = cfg.analytics.reporting.samples;
		};
		server.addr = cfg.server.addr;
		cache = {
		allow-delete-verb = cfg.cache.allowDeleteVerb;
		allow-put-verb = cfg.cache.allowPutVerb;
		hostname = cfg.cache.hostName;
		database-url = cfg.cache.databaseURL;
		database.pool = {
		max-open-conns = cfg.cache.database.pool.maxOpenConns;
		max-idle-conns = cfg.cache.database.pool.maxIdleConns;
		};
		max-size = cfg.cache.maxSize;
		lru = {
		schedule = cfg.cache.lru.schedule;
		timezone = cfg.cache.lru.scheduleTimeZone;
		};
		sign-narinfo = cfg.cache.signNarinfo;
		storage =
		if cfg.cache.storage.s3 != null then
		{
		s3 = {
		bucket = cfg.cache.storage.s3.bucket;
		endpoint = cfg.cache.storage.s3.endpoint;
		region = cfg.cache.storage.s3.region;
		force-path-style = cfg.cache.storage.s3.forcePathStyle;
		};
		}
		else
		{
		local = cfg.cache.storage.local;
		};
		temp-path = cfg.cache.tempPath;
		netrc-file = cfg.netrcFile;
		upstream = {
		urls = cfg.cache.upstream.urls;
		public-keys = cfg.cache.upstream.publicKeys;
		dialer-timeout = cfg.cache.upstream.dialerTimeout;
		response-header-timeout = cfg.cache.upstream.responseHeaderTimeout;
		};
		lock = {
		backend = cfg.cache.lock.backend;
		redis.key-prefix = cfg.cache.lock.redisKeyPrefix;
		postgres.key-prefix = cfg.cache.lock.postgresKeyPrefix;
		download-lock-ttl = cfg.cache.lock.downloadTTL;
		lru-lock-ttl = cfg.cache.lock.lruTTL;
		retry = {
		max-attempts = cfg.cache.lock.retry.maxAttempts;
		initial-delay = cfg.cache.lock.retry.initialDelay;
		max-delay = cfg.cache.lock.retry.maxDelay;
		jitter = cfg.cache.lock.retry.jitter;
		};
		allow-degraded-mode = cfg.cache.lock.allowDegradedMode;
		};
		redis = lib.optionalAttrs (cfg.cache.redis != null) {
		addrs = cfg.cache.redis.addresses;
		db = cfg.cache.redis.database;
		username = cfg.cache.redis.username;
		use-tls = cfg.cache.redis.useTLS;
		pool-size = cfg.cache.redis.poolSize;
		};
		};
		};

		configFile = pkgs.writeText "ncps-config.json" (
		builtins.toJSON (
		lib.filterAttrsRecursive (_: v: v != null && v != { } && v != "" && v != [ ]) settings
		)
		);

		isSqlite = cfg.cache.databaseURL != null && lib.strings.hasPrefix "sqlite:" cfg.cache.databaseURL;
		@@ -136,7 +157,8 @@ in
		services.ncps = {
		enable = lib.mkEnableOption "ncps: Nix binary cache proxy service implemented in Go";

		analytics.reporting.enable = lib.mkOption {
		analytics.reporting = {
		enable = lib.mkOption {
		type = lib.types.bool;
		default = true;
		description = ''
		@@ -144,6 +166,9 @@ in
		'';
		};

		samples = lib.mkEnableOption "Enable printing the analytics samples to stdout. This is useful for debugging and verification purposes only.";
		};

		package = lib.mkPackageOption pkgs "ncps" { };

		openTelemetry = {
		@@ -208,6 +233,28 @@ in
		'';
		};

		database = {
		pool = {
		maxOpenConns = lib.mkOption {
		type = lib.types.int;
		default = 0;
		description = ''
		Maximum number of open connections to the database (0 = use
		database-specific defaults).
		'';
		};

		maxIdleConns = lib.mkOption {
		type = lib.types.int;
		default = 0;
		description = ''
		Maximum number of idle connections in the pool (0 = use
		database-specific defaults).
		'';
		};
		};
		};

		lru = {
		schedule = lib.mkOption {
		type = lib.types.nullOr lib.types.str;
		@@ -233,7 +280,8 @@ in
		};
		};

		lock.backend = lib.mkOption {
		lock = {
		backend = lib.mkOption {
		type = lib.types.enum [
		"local"
		"redis"
		@@ -242,10 +290,99 @@ in
		default = "local";
		description = ''
		Lock backend to use: 'local' (single instance), 'redis'
		(distributed), or 'postgres' (distributed, requires PostgreSQL).
		(distributed), 'postgres' (distributed, requires PostgreSQL).

		Advisory Locks and Connection Pools: If you use PostgreSQL as
		your distributed lock backend, each active lock consumes a
		dedicated connection from the pool. A single request can consume
		up to 3 connections simultaneously.

		To avoid deadlocks under concurrent load, ensure
		{option}`services.ncps.cache.database.pool.maxOpenConns` is
		significantly higher than your expected concurrency (at least
		50-100 is recommended).
		'';
		};

		redisKeyPrefix = lib.mkOption {
		type = lib.types.str;
		default = "ncps:lock:";
		description = ''
		Prefix for all Redis lock keys (only used when Redis is
		configured).
		'';
		};

		postgresKeyPrefix = lib.mkOption {
		type = lib.types.str;
		default = "ncps:lock:";
		description = ''
		Prefix for all PostgreSQL advisory lock keys (only used when
		PostgreSQL is configured as lock backend).
		'';
		};

		downloadTTL = lib.mkOption {
		type = lib.types.str;
		default = "5m0s";
		description = ''
		TTL for download locks (per-hash locks).
		'';
		};

		lruTTL = lib.mkOption {
		type = lib.types.str;
		default = "30m0s";
		description = ''
		TTL for LRU lock (global exclusive lock).
		'';
		};

		retry = {
		maxAttempts = lib.mkOption {
		type = lib.types.int;
		default = 3;
		description = ''
		Maximum number of retry attempts for distributed locks.
		'';
		};

		initialDelay = lib.mkOption {
		type = lib.types.str;
		default = "100ms";
		description = ''
		Initial retry delay for distributed locks.
		'';
		};

		maxDelay = lib.mkOption {
		type = lib.types.str;
		default = "2s";
		description = ''
		Maximum retry delay for distributed locks (exponential backoff
		caps at this).
		'';
		};

		jitter = lib.mkOption {
		type = lib.types.bool;
		default = true;
		description = ''
		Enable jitter in retry delays to prevent thundering herd.
		'';
		};
		};

		allowDegradedMode = lib.mkOption {
		type = lib.types.bool;
		default = false;
		description = ''
		Allow falling back to local locks if Redis is unavailable (WARNING:
		breaks HA guarantees).
		'';
		};
		};

		maxSize = lib.mkOption {
		type = lib.types.nullOr lib.types.str;
		default = null;
		@@ -555,7 +692,7 @@ in

		serviceConfig = lib.mkMerge [
		{
		ExecStart = "${ncpsWrapper} ${globalFlags} serve ${serveFlags}";
		ExecStart = "${ncpsWrapper} serve";
		User = "ncps";
		Group = "ncps";
		Restart = "on-failure";

nixos/tests/all-tests.nix

+2 −1

Original line number	Diff line number	Diff line
		@@ -1039,7 +1039,8 @@ in
		imports = [ ./ncps.nix ];
		defaults.services.ncps.cache.storage.local = "/path/to/ncps";
		};
		ncps-ha = runTest ./ncps-ha.nix;
		ncps-ha-pg = runTest ./ncps-ha-pg.nix;
		ncps-ha-pg-redis = runTest ./ncps-ha-pg-redis.nix;
		ndppd = runTest ./ndppd.nix;
		nebula-lighthouse-service = runTest ./nebula-lighthouse-service.nix;
		nebula.connectivity = runTest ./nebula/connectivity.nix;

nixos/tests/ncps-ha.nix→nixos/tests/ncps-ha-pg-redis.nix

+0 −0

File moved.

View file

nixos/tests/ncps-ha-pg.nix

0 → 100644

+196 −0

Original line number	Diff line number	Diff line
		{
		lib,
		pkgs,
		...
		}:
		let
		# s3 creds
		bucket = "ncps";
		region = "us-west-1";
		accessKey = builtins.toFile "minio-access-key" "easy-key";
		secretKey = builtins.toFile "minio-secret-key" "easy-secret";

		# pg creds
		postgresPassword = "easypwd";

		initMinio = pkgs.writeShellScriptBin "init-minio.sh" ''
		set -euo pipefail

		mc alias set local "http://127.0.0.1:9000" minioadmin minioadmin
		mc mb local/${bucket}
		mc admin user svcacct add --access-key "$(cat ${accessKey})" --secret-key "$(cat ${secretKey})" local minioadmin
		'';

		ncpsAttrs = hostname: {
		services.ncps = {
		enable = true;

		analytics.reporting.enable = false;

		cache = {
		hostName = hostname;

		databaseURL = "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable";

		lock.backend = "postgres";

		secretKeyPath = builtins.toString (
		pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA=="
		);

		storage.s3 = {
		inherit bucket region;

		endpoint = "http://minio:9000";

		accessKeyIdPath = accessKey;
		secretAccessKeyPath = secretKey;
		};

		upstream = {
		urls = [ "http://harmonia:5000" ];
		publicKeys = [
		"cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo="
		];
		};
		};
		};

		networking.firewall.allowedTCPPorts = [ 8501 ];
		};
		in
		{
		name = "ncps-storage-s3";
		meta = with lib.maintainers; {
		maintainers = [
		aciceri
		kalbasit
		];
		};

		nodes = {
		client0 = {
		nix.settings = {
		substituters = lib.mkForce [ "http://ncps0:8501" ];
		trusted-public-keys = lib.mkForce [
		"ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg="
		];
		};
		};

		client1 = {
		nix.settings = {
		substituters = lib.mkForce [ "http://ncps1:8501" ];
		trusted-public-keys = lib.mkForce [
		"ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg="
		];
		};
		};

		harmonia = {
		services.harmonia = {
		enable = true;
		signKeyPaths = [
		(pkgs.writeText "cache-key" "cache.example.com-1:9FhO0w+7HjZrhvmzT1VlAZw4OSAlFGTgC24Seg3tmPl4gZBdwZClzTTHr9cVzJpwsRSYLTu7hEAQe3ljy92CWg==")
		];
		settings.priority = 35;
		};

		networking.firewall.allowedTCPPorts = [ 5000 ];
		system.extraDependencies = [ pkgs.emptyFile ];
		};

		minio = {
		services.minio = {
		inherit region;

		enable = true;
		};

		networking.firewall.allowedTCPPorts = [ 9000 ];
		environment.systemPackages = [
		pkgs.minio-client
		initMinio
		];
		};

		ncps0 = lib.mkMerge [
		(ncpsAttrs "ncps0")
		{
		services.ncps.cache.databaseURL = lib.mkForce null;
		services.ncps.cache.databaseURLFile = builtins.toFile "db-url" "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable";
		}
		];
		ncps1 = ncpsAttrs "ncps1";

		postgres = {
		services.postgresql = {
		enable = true;
		enableTCPIP = true;
		authentication = ''
		host all all all scram-sha-256
		'';
		initialScript = pkgs.writeText "init-postgres.sql" ''
		CREATE DATABASE "ncps" WITH ENCODING = 'UTF8';
		CREATE ROLE "ncps" WITH LOGIN PASSWORD '${
		builtins.replaceStrings [ "'" ] [ "''" ] postgresPassword
		}';
		ALTER DATABASE "ncps" OWNER TO "ncps";
		'';
		};

		networking.firewall.allowedTCPPorts = [ 5432 ];
		};
		};

		testScript =
		{ nodes, ... }:
		let
		narinfoName =
		(lib.strings.removePrefix "/nix/store/" (
		lib.strings.removeSuffix "-empty-file" pkgs.emptyFile.outPath
		))
		+ ".narinfo";

		narinfoNameChars = lib.strings.stringToCharacters narinfoName;

		narinfoPath = lib.concatStringsSep "/" [
		(builtins.head nodes.minio.services.minio.dataDir)
		bucket
		"store/narinfo"
		(lib.lists.elemAt narinfoNameChars 0)
		((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1))
		narinfoName
		"xl.meta"
		];
		in
		''
		harmonia.start()
		minio.start()
		postgres.start()

		minio.wait_for_unit("minio.service")

		minio.wait_until_succeeds("init-minio.sh")

		postgres.wait_for_unit("postgresql.service")

		start_all()

		harmonia.wait_for_unit("harmonia.service")

		ncps0.wait_for_unit("ncps.service")
		ncps1.wait_for_unit("ncps.service")

		client0.wait_until_succeeds("curl -f http://ncps0:8501/ \| grep '\"hostname\":\"${toString nodes.ncps0.services.ncps.cache.hostName}\"' >&2")
		client1.wait_until_succeeds("curl -f http://ncps1:8501/ \| grep '\"hostname\":\"${toString nodes.ncps1.services.ncps.cache.hostName}\"' >&2")

		client0.succeed("cat /etc/nix/nix.conf >&2")
		client0.succeed("nix-store --realise ${pkgs.emptyFile}")

		client1.succeed("cat /etc/nix/nix.conf >&2")
		client1.succeed("nix-store --realise ${pkgs.emptyFile}")

		minio.succeed("cat ${narinfoPath} >&2")
		'';
		}