Merge release-23.11 into staging-next-23.11

           else args';

    # TODO: deprecate args.rustc in favour of args.rust after 23.05 is EOL.

    rust = assert !(args ? rust && args ? rustc); args.rust or args.rustc or {};

    rust = args.rust or args.rustc or {};

    final = {

      # Prefer to parse `config` as it is strictly more informative.

      # TODO: remove after 23.05 is EOL, with an error pointing to the rust.* attrs.

      rustc = args.rustc or {};

      rust = rust // {

        # Once args.rustc.platform.target-family is deprecated and

        # removed, there will no longer be any need to modify any

        # values from args.rust.platform, so we can drop all the

        # "args ? rust" etc. checks, and merge args.rust.platform in

        # /after/.

        platform = rust.platform or {} // {

          # https://doc.rust-lang.org/reference/conditional-compilation.html#target_arch

          arch =

            /**/ if rust ? platform then rust.platform.arch

            else if final.isAarch32 then "arm"

            else if final.isMips64  then "mips64"     # never add "el" suffix

            else if final.isPower64 then "powerpc64"  # never add "le" suffix

            else final.parsed.cpu.name;

          # https://doc.rust-lang.org/reference/conditional-compilation.html#target_os

          os =

            /**/ if rust ? platform then rust.platform.os or "none"

            else if final.isDarwin then "macos"

            else final.parsed.kernel.name;

          # https://doc.rust-lang.org/reference/conditional-compilation.html#target_family

          target-family =

            /**/ if args ? rust.platform.target-family then args.rust.platform.target-family

            else if args ? rustc.platform.target-family

            then

              (

                # Since https://github.com/rust-lang/rust/pull/84072

                # `target-family` is a list instead of single value.

                let

                  f = args.rustc.platform.target-family;

                in

                  if builtins.isList f then f else [ f ]

              )

            else lib.optional final.isUnix "unix"

                 ++ lib.optional final.isWindows "windows";

          # https://doc.rust-lang.org/reference/conditional-compilation.html#target_vendor

          vendor = let

            inherit (final.parsed) vendor;

          in rust.platform.vendor or {

            "w64" = "pc";

          }.${vendor.name} or vendor.name;

        };

        # The name of the rust target, even if it is custom. Adjustments are

        # because rust has slightly different naming conventions than we do.

        rustcTarget = let

          inherit (final.parsed) cpu kernel abi;

          cpu_ = rust.platform.arch or {

            "armv7a" = "armv7";

            "armv7l" = "armv7";

            "armv6l" = "arm";

            "armv5tel" = "armv5te";

            "riscv64" = "riscv64gc";

          }.${cpu.name} or cpu.name;

          vendor_ = final.rust.platform.vendor;

        in rust.config

          or "${cpu_}-${vendor_}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}";

        # The name of the rust target if it is standard, or the json file

        # containing the custom target spec.

        rustcTargetSpec =

          /**/ if rust ? platform

          then builtins.toFile (final.rust.rustcTarget + ".json") (builtins.toJSON rust.platform)

          else final.rust.rustcTarget;

        # The name of the rust target if it is standard, or the

        # basename of the file containing the custom target spec,

        # without the .json extension.

        #

        # This is the name used by Cargo for target subdirectories.

        cargoShortTarget =

          lib.removeSuffix ".json" (baseNameOf "${final.rust.rustcTargetSpec}");

        # When used as part of an environment variable name, triples are

        # uppercased and have all hyphens replaced by underscores:

        #

        # https://github.com/rust-lang/cargo/pull/9169

        # https://github.com/rust-lang/cargo/issues/8285#issuecomment-634202431

        cargoEnvVarTarget =

          lib.strings.replaceStrings ["-"] ["_"]

            (lib.strings.toUpper final.rust.cargoShortTarget);

        # True if the target is no_std

        # https://github.com/rust-lang/rust/blob/2e44c17c12cec45b6a682b1e53a04ac5b5fcc9d2/src/bootstrap/config.rs#L415-L421

        isNoStdTarget =

          builtins.any (t: lib.hasInfix t final.rust.rustcTarget) ["-none" "nvptx" "switch" "-uefi"];

      };

      linuxArch =

        if final.isAarch32 then "arm"

        else if final.isAarch64 then "arm64"

    }) // mapAttrs (n: v: v final.parsed) inspect.predicates

      // mapAttrs (n: v: v final.gcc.arch or "default") architectures.predicates

      // args;

      // args // {

        rust = rust // {

          # Once args.rustc.platform.target-family is deprecated and

          # removed, there will no longer be any need to modify any

          # values from args.rust.platform, so we can drop all the

          # "args ? rust" etc. checks, and merge args.rust.platform in

          # /after/.

          platform = rust.platform or {} // {

            # https://doc.rust-lang.org/reference/conditional-compilation.html#target_arch

            arch =

              /**/ if rust ? platform then rust.platform.arch

              else if final.isAarch32 then "arm"

              else if final.isMips64  then "mips64"     # never add "el" suffix

              else if final.isPower64 then "powerpc64"  # never add "le" suffix

              else final.parsed.cpu.name;

            # https://doc.rust-lang.org/reference/conditional-compilation.html#target_os

            os =

              /**/ if rust ? platform then rust.platform.os or "none"

              else if final.isDarwin then "macos"

              else final.parsed.kernel.name;

            # https://doc.rust-lang.org/reference/conditional-compilation.html#target_family

            target-family =

              /**/ if args ? rust.platform.target-family then args.rust.platform.target-family

              else if args ? rustc.platform.target-family

              then

                (

                  # Since https://github.com/rust-lang/rust/pull/84072

                  # `target-family` is a list instead of single value.

                  let

                    f = args.rustc.platform.target-family;

                  in

                    if builtins.isList f then f else [ f ]

                )

              else lib.optional final.isUnix "unix"

                   ++ lib.optional final.isWindows "windows";

            # https://doc.rust-lang.org/reference/conditional-compilation.html#target_vendor

            vendor = let

              inherit (final.parsed) vendor;

            in rust.platform.vendor or {

              "w64" = "pc";

            }.${vendor.name} or vendor.name;

          };

          # The name of the rust target, even if it is custom. Adjustments are

          # because rust has slightly different naming conventions than we do.

          rustcTarget = let

            inherit (final.parsed) cpu kernel abi;

            cpu_ = rust.platform.arch or {

              "armv7a" = "armv7";

              "armv7l" = "armv7";

              "armv6l" = "arm";

              "armv5tel" = "armv5te";

              "riscv64" = "riscv64gc";

            }.${cpu.name} or cpu.name;

            vendor_ = final.rust.platform.vendor;

          in rust.config

            or "${cpu_}-${vendor_}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}";

          # The name of the rust target if it is standard, or the json file

          # containing the custom target spec.

          rustcTargetSpec = rust.rustcTargetSpec or (

            /**/ if rust ? platform

            then builtins.toFile (final.rust.rustcTarget + ".json") (builtins.toJSON rust.platform)

            else final.rust.rustcTarget);

          # The name of the rust target if it is standard, or the

          # basename of the file containing the custom target spec,

          # without the .json extension.

          #

          # This is the name used by Cargo for target subdirectories.

          cargoShortTarget =

            lib.removeSuffix ".json" (baseNameOf "${final.rust.rustcTargetSpec}");

          # When used as part of an environment variable name, triples are

          # uppercased and have all hyphens replaced by underscores:

          #

          # https://github.com/rust-lang/cargo/pull/9169

          # https://github.com/rust-lang/cargo/issues/8285#issuecomment-634202431

          cargoEnvVarTarget =

            lib.strings.replaceStrings ["-"] ["_"]

              (lib.strings.toUpper final.rust.cargoShortTarget);

          # True if the target is no_std

          # https://github.com/rust-lang/rust/blob/2e44c17c12cec45b6a682b1e53a04ac5b5fcc9d2/src/bootstrap/config.rs#L415-L421

          isNoStdTarget =

            builtins.any (t: lib.hasInfix t final.rust.rustcTarget) ["-none" "nvptx" "switch" "-uefi"];

        };

      };

  in assert final.useAndroidPrebuilt -> final.isAndroid;

     assert lib.foldl

       (pass: { assertion, message }:

    githubId = 424946;

    name = "James Earl Douglas";

  };

  EBADBEEF = {

    name = "EBADBEEF";

    email = "errno@ebadf.com";

    github = "EBADBEEF";

    githubId = 4167946;

  };

  ebbertd = {

    email = "daniel@ebbert.nrw";

    github = "ebbertd";

- The binary of the package `cloud-sql-proxy` has changed from `cloud_sql_proxy` to `cloud-sql-proxy`.

- The module `services.apache-kafka` was largely rewritten and has certain breaking changes. To be precise, this means that the following things have changed:

  - Most settings have been migrated to [services.apache-kafka.settings](#opt-services.apache-kafka.settings).

    - Care must be taken when adapting an existing cluster to these changes, see [](#module-services-apache-kafka-migrating-to-settings).

  - By virtue of being less opinionated, it is now possible to use the module to run Apache Kafka in KRaft mode instead of Zookeeper mode.

    - [A few options](#module-services-apache-kafka-kraft) have been added to assist in this mode.

- Garage has been upgraded to 0.9.x. `services.garage.package` now needs to be explicitly set, so version upgrades can be done in a controlled fashion. For this, we expose `garage_x_y` attributes which can be set here.

- `voms` and `xrootd` now moves the `$out/etc` content to the `$etc` output instead of `$out/etc.orig`, when input argument `externalEtc` is not `null`.

- The `woodpecker-*` CI packages have been updated to 1.0.0. This release is wildly incompatible with the 0.15.X versions that were previously packaged. Please read [upstream's documentation](https://woodpecker-ci.org/docs/next/migrations#100) to learn how to update your CI configurations.

- Meilisearch was updated from 1.3.1 to 1.5.0. The update has breaking changes about backslashes and filtering. See the [release announcement](https://blog.meilisearch.com/v1-4-release/) for more information.

- The Caddy module gained a new option named `services.caddy.enableReload` which is enabled by default. It allows reloading the service instead of restarting it, if only a config file has changed. This option must be disabled if you have turned off the [Caddy admin API](https://caddyserver.com/docs/caddyfile/options#admin). If you keep this option enabled, you should consider setting [`grace_period`](https://caddyserver.com/docs/caddyfile/options#grace-period) to a non-infinite value to prevent Caddy from delaying the reload indefinitely.

- mdraid support is now optional. This reduces initramfs size and prevents the potentially undesired automatic detection and activation of software RAID pools. It is disabled by default in new configurations (determined by `stateVersion`), but the appropriate settings will be generated by `nixos-generate-config` when installing to a software RAID device, so the standard installation procedure should be unaffected. If you have custom configs relying on mdraid, ensure that you use `stateVersion` correctly or set `boot.swraid.enable` manually. On systems with an updated `stateVersion` we now also emit warnings if `mdadm.conf` does not contain the minimum required configuration necessary to run the dynamically enabled monitoring daemons.

  imports =

    [

      # Include the default lxd configuration.

      "${modulesPath}/modules/virtualisation/lxc-container.nix"

      "${modulesPath}/virtualisation/lxc-container.nix"

      # Include the container-specific autogenerated configuration.

      ./lxd.nix

    ];

let

  cfg = config.services.apache-kafka;

  serverProperties =

    if cfg.serverProperties != null then

      cfg.serverProperties

    else

      ''

        # Generated by nixos

        broker.id=${toString cfg.brokerId}

        port=${toString cfg.port}

        host.name=${cfg.hostname}

        log.dirs=${concatStringsSep "," cfg.logDirs}

        zookeeper.connect=${cfg.zookeeper}

        ${toString cfg.extraProperties}

      '';

  serverConfig = pkgs.writeText "server.properties" serverProperties;

  logConfig = pkgs.writeText "log4j.properties" cfg.log4jProperties;

  # The `javaProperties` generator takes care of various escaping rules and

  # generation of the properties file, but we'll handle stringly conversion

  # ourselves in mkPropertySettings and stringlySettings, since we know more

  # about the specifically allowed format eg. for lists of this type, and we

  # don't want to coerce-downsample values to str too early by having the

  # coercedTypes from javaProperties directly in our NixOS option types.

  #

  # Make sure every `freeformType` and any specific option type in `settings` is

  # supported here.

  mkPropertyString = let

    render = {

      bool = boolToString;

      int = toString;

      list = concatMapStringsSep "," mkPropertyString;

      string = id;

    };

  in

    v: render.${builtins.typeOf v} v;

  stringlySettings = mapAttrs (_: mkPropertyString)

    (filterAttrs (_: v:  v != null) cfg.settings);

  generator = (pkgs.formats.javaProperties {}).generate;

in {

  options.services.apache-kafka = {

    enable = mkOption {

      description = lib.mdDoc "Whether to enable Apache Kafka.";

      default = false;

      type = types.bool;

    enable = mkEnableOption (lib.mdDoc "Apache Kafka event streaming broker");

    settings = mkOption {

      description = lib.mdDoc ''

        [Kafka broker configuration](https://kafka.apache.org/documentation.html#brokerconfigs)

        {file}`server.properties`.

        Note that .properties files contain mappings from string to string.

        Keys with dots are NOT represented by nested attrs in these settings,

        but instead as quoted strings (ie. `settings."broker.id"`, NOT

        `settings.broker.id`).

     '';

      type = types.submodule {

        freeformType = with types; let

          primitive = oneOf [bool int str];

        in lazyAttrsOf (nullOr (either primitive (listOf primitive)));

        options = {

          "broker.id" = mkOption {

            description = lib.mdDoc "Broker ID. -1 or null to auto-allocate in zookeeper mode.";

            default = null;

            type = with types; nullOr int;

          };

    brokerId = mkOption {

      description = lib.mdDoc "Broker ID.";

      default = -1;

      type = types.int;

          "log.dirs" = mkOption {

            description = lib.mdDoc "Log file directories.";

            # Deliberaly leave out old default and use the rewrite opportunity

            # to have users choose a safer value -- /tmp might be volatile and is a

            # slightly scary default choice.

            # default = [ "/tmp/apache-kafka" ];

            type = with types; listOf path;

          };

    port = mkOption {

      description = lib.mdDoc "Port number the broker should listen on.";

      default = 9092;

      type = types.port;

          "listeners" = mkOption {

            description = lib.mdDoc ''

              Kafka Listener List.

              See [listeners](https://kafka.apache.org/documentation/#brokerconfigs_listeners).

            '';

            type = types.listOf types.str;

            default = [ "PLAINTEXT://localhost:9092" ];

          };

        };

      };

    };

    hostname = mkOption {

      description = lib.mdDoc "Hostname the broker should bind to.";

      default = "localhost";

      type = types.str;

    clusterId = mkOption {

      description = lib.mdDoc ''

        KRaft mode ClusterId used for formatting log directories. Can be generated with `kafka-storage.sh random-uuid`

      '';

      type = with types; nullOr str;

      default = null;

    };

    logDirs = mkOption {

      description = lib.mdDoc "Log file directories";

      default = [ "/tmp/kafka-logs" ];

      type = types.listOf types.path;

    configFiles.serverProperties = mkOption {

      description = lib.mdDoc ''

        Kafka server.properties configuration file path.

        Defaults to the rendered `settings`.

      '';

      type = types.path;

    };

    zookeeper = mkOption {

      description = lib.mdDoc "Zookeeper connection string";

      default = "localhost:2181";

      type = types.str;

    configFiles.log4jProperties = mkOption {

      description = lib.mdDoc "Kafka log4j property configuration file path";

      type = types.path;

      default = pkgs.writeText "log4j.properties" cfg.log4jProperties;

      defaultText = ''pkgs.writeText "log4j.properties" cfg.log4jProperties'';

    };

    extraProperties = mkOption {

      description = lib.mdDoc "Extra properties for server.properties.";

      type = types.nullOr types.lines;

      default = null;

    formatLogDirs = mkOption {

      description = lib.mdDoc ''

        Whether to format log dirs in KRaft mode if all log dirs are

        unformatted, ie. they contain no meta.properties.

      '';

      type = types.bool;

      default = false;

    };

    serverProperties = mkOption {

    formatLogDirsIgnoreFormatted = mkOption {

      description = lib.mdDoc ''

        Complete server.properties content. Other server.properties config

        options will be ignored if this option is used.

        Whether to ignore already formatted log dirs when formatting log dirs,

        instead of failing. Useful when replacing or adding disks.

      '';

      type = types.nullOr types.lines;

      default = null;

      type = types.bool;

      default = false;

    };

    log4jProperties = mkOption {

      defaultText = literalExpression "pkgs.apacheKafka.passthru.jre";

      type = types.package;

    };

  };

  config = mkIf cfg.enable {

  imports = [

    (mkRenamedOptionModule

      [ "services" "apache-kafka" "brokerId" ]

      [ "services" "apache-kafka" "settings" ''broker.id'' ])

    (mkRenamedOptionModule

      [ "services" "apache-kafka" "logDirs" ]

      [ "services" "apache-kafka" "settings" ''log.dirs'' ])

    (mkRenamedOptionModule

      [ "services" "apache-kafka" "zookeeper" ]

      [ "services" "apache-kafka" "settings" ''zookeeper.connect'' ])

    (mkRemovedOptionModule [ "services" "apache-kafka" "port" ]

      "Please see services.apache-kafka.settings.listeners and its documentation instead")

    (mkRemovedOptionModule [ "services" "apache-kafka" "hostname" ]

      "Please see services.apache-kafka.settings.listeners and its documentation instead")

    (mkRemovedOptionModule [ "services" "apache-kafka" "extraProperties" ]

      "Please see services.apache-kafka.settings and its documentation instead")

    (mkRemovedOptionModule [ "services" "apache-kafka" "serverProperties" ]

      "Please see services.apache-kafka.settings and its documentation instead")

  ];

    environment.systemPackages = [cfg.package];

  config = mkIf cfg.enable {

    services.apache-kafka.configFiles.serverProperties = generator "server.properties" stringlySettings;

    users.users.apache-kafka = {

      isSystemUser = true;

      group = "apache-kafka";

      description = "Apache Kafka daemon user";

      home = head cfg.logDirs;

    };

    users.groups.apache-kafka = {};

    systemd.tmpfiles.rules = map (logDir: "d '${logDir}' 0700 apache-kafka - - -") cfg.logDirs;

    systemd.tmpfiles.rules = map (logDir: "d '${logDir}' 0700 apache-kafka - - -") cfg.settings."log.dirs";

    systemd.services.apache-kafka = {

      description = "Apache Kafka Daemon";

      wantedBy = [ "multi-user.target" ];

      after = [ "network.target" ];

      preStart = mkIf cfg.formatLogDirs

        (if cfg.formatLogDirsIgnoreFormatted then ''

          ${cfg.package}/bin/kafka-storage.sh format -t "${cfg.clusterId}" -c ${cfg.configFiles.serverProperties} --ignore-formatted

        '' else ''

          if ${concatMapStringsSep " && " (l: ''[ ! -f "${l}/meta.properties" ]'') cfg.settings."log.dirs"}; then

            ${cfg.package}/bin/kafka-storage.sh format -t "${cfg.clusterId}" -c ${cfg.configFiles.serverProperties}

          fi

        '');

      serviceConfig = {

        ExecStart = ''

          ${cfg.jre}/bin/java \

            -cp "${cfg.package}/libs/*" \

            -Dlog4j.configuration=file:${logConfig} \

            -Dlog4j.configuration=file:${cfg.configFiles.log4jProperties} \

            ${toString cfg.jvmOptions} \

            kafka.Kafka \

            ${serverConfig}

            ${cfg.configFiles.serverProperties}

        '';

        User = "apache-kafka";

        SuccessExitStatus = "0 143";

      };

    };

  };

  meta.doc = ./kafka.md;

  meta.maintainers = with lib.maintainers; [

    srhb

  ];

}