nixos/frr: refactor (#327099)

- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable).

- [FRRouting](https://frrouting.org/), a popular suite of Internet routing protocol daemons (BGP, BFD, OSPF, IS-IS, VRRP and others). Available as [services.frr](#opt-services.frr.babel.enable).

- [FRRouting](https://frrouting.org/), a popular suite of Internet routing protocol daemons (BGP, BFD, OSPF, IS-IS, VRRP and others). Available as [services.frr](#opt-services.frr.babeld.enable).

- [Grafana Mimir](https://grafana.com/oss/mimir/), an open source, horizontally scalable, highly available, multi-tenant, long-term storage for Prometheus. Available as [services.mimir](#opt-services.mimir.enable).

- `ceph` has been upgraded to v19. See the [Ceph "squid" release notes](https://docs.ceph.com/en/latest/releases/squid/#v19-2-0-squid) for details and recommended upgrade procedure.

- `services.frr` has been refactored to use upstream service scripts. The per-daemon configurations

  have been removed in favour of an `integrated-vtysh-config` style config. The daemon submodules

  now use the daemon name (e.g. `ospfd`) instead of the protocol name (`ospf`). The daemons `zebra`,

  `mgmtd` and `staticd` are always enabled if a config is present. The `vtyListenAddress` and

  `vtyListenPort` options have been removed; use `options` or `extraOptions` instead, respectively.

- `opencv2` and `opencv3` have been removed, as they are obsolete and

  were not used by any other package. External users are encouraged to

  migrate to OpenCV 4.

{ config, lib, pkgs, ... }:

let

  cfg = config.services.frr;

  services = [

    "static"

  daemons = [

    "bgpd"

    "ospfd"

    "ospf6d"

    "ripd"

    "ripngd"

    "isisd"

    "pimd"

    "pim6d"

    "ldpd"

    "nhrpd"

    "eigrpd"

    "babeld"

    "sharpd"

    "pbrd"

    "bfdd"

    "fabricd"

    "vrrpd"

    "pathd"

  ];

  daemonDefaultOptions = {

    zebra = "-A 127.0.0.1 -s 90000000";

    mgmtd = "-A 127.0.0.1";

    bgpd = "-A 127.0.0.1";

    ospfd = "-A 127.0.0.1";

    ospf6d = "-A ::1";

    ripd = "-A 127.0.0.1";

    ripngd = "-A ::1";

    isisd = "-A 127.0.0.1";

    pimd = "-A 127.0.0.1";

    pim6d = "-A ::1";

    ldpd = "-A 127.0.0.1";

    nhrpd = "-A 127.0.0.1";

    eigrpd = "-A 127.0.0.1";

    babeld = "-A 127.0.0.1";

    sharpd = "-A 127.0.0.1";

    pbrd = "-A 127.0.0.1";

    staticd = "-A 127.0.0.1";

    bfdd = "-A 127.0.0.1";

    fabricd = "-A 127.0.0.1";

    vrrpd = "-A 127.0.0.1";

    pathd = "-A 127.0.0.1";

  };

  renamedServices = [

    "bgp"

    "ospf"

    "ospf6"

    "fabric"

  ];

  allServices = services ++ [ "zebra" "mgmt" ];

  obsoleteServices = renamedServices ++ [ "static" "mgmt" "zebra" ];

  allDaemons = builtins.attrNames daemonDefaultOptions;

  isEnabled = service: cfg.${service}.enable;

  daemonName = service: if service == "zebra" then service else "${service}d";

  daemonLine = d: "${d}=${if isEnabled d then "yes" else "no"}";

  configFile = service:

    let

      scfg = cfg.${service};

    in

      if scfg.configFile != null then scfg.configFile

      else pkgs.writeText "${daemonName service}.conf"

        ''

          ! FRR ${daemonName service} configuration

  configFile =

    if cfg.configFile != null then

      cfg.configFile

    else

      pkgs.writeText "frr.conf" ''

        ! FRR configuration

        !

        hostname ${config.networking.hostName}

        log syslog

        service password-encryption

        service integrated-vtysh-config

        !

          ${scfg.config}

        ${cfg.config}

        !

        end

      '';

  serviceOptions = service:

  serviceOptions =

    service:

    {

      enable = lib.mkEnableOption "the FRR ${lib.toUpper service} routing protocol";

      configFile = lib.mkOption {

        type = lib.types.nullOr lib.types.path;

        default = null;

        example = "/etc/frr/${daemonName service}.conf";

        description = ''

          Configuration file to use for FRR ${daemonName service}.

          By default the NixOS generated files are used.

        '';

      };

      config = lib.mkOption {

        type = lib.types.lines;

        default = "";

        example =

          let

            examples = {

              rip = ''

                router rip

                  network 10.0.0.0/8

              '';

              ospf = ''

                router ospf

                  network 10.0.0.0/8 area 0

              '';

              bgp = ''

                router bgp 65001

                  neighbor 10.0.0.1 remote-as 65001

              '';

            };

          in

            examples.${service} or "";

        description = ''

          ${daemonName service} configuration statements.

        '';

      };

      vtyListenAddress = lib.mkOption {

        type = lib.types.str;

        default = "localhost";

        description = ''

          Address to bind to for the VTY interface.

        '';

      };

      vtyListenPort = lib.mkOption {

        type = lib.types.nullOr lib.types.int;

        default = null;

      options = lib.mkOption {

        type = lib.types.listOf lib.types.str;

        default = [ daemonDefaultOptions.${service} ];

        description = ''

          TCP Port to bind to for the VTY interface.

          Options for the FRR ${service} daemon.

        '';

      };

      extraOptions = lib.mkOption {

        type = lib.types.listOf lib.types.str;

        default = [ ];

        description = ''

          Extra options for the daemon.

          Extra options to be appended to the FRR ${service} daemon options.

        '';

      };

    };

    }

    // (if (builtins.elem service daemons) then { enable = lib.mkEnableOption "FRR ${service}"; } else { });

in

{

  ###### interface

  imports = [

  imports =

    [

      {

        options.services.frr = {

        zebra = (serviceOptions "zebra") // {

          enable = lib.mkOption {

            type = lib.types.bool;

            default = lib.any isEnabled services;

          configFile = lib.mkOption {

            type = lib.types.nullOr lib.types.path;

            default = null;

            example = "/etc/frr/frr.conf";

            description = ''

              Whether to enable the Zebra routing manager.

              The Zebra routing manager is automatically enabled

              if any routing protocols are configured.

              Configuration file to use for FRR.

              By default the NixOS generated files are used.

            '';

          };

        };

        mgmt = (serviceOptions "mgmt") // {

          enable = lib.mkOption {

            type = lib.types.bool;

            default = isEnabled "static";

            defaultText = lib.literalExpression "config.services.frr.static.enable";

          config = lib.mkOption {

            type = lib.types.lines;

            default = "";

            example = ''

              router rip

                network 10.0.0.0/8

              router ospf

                network 10.0.0.0/8 area 0

              router bgp 65001

                neighbor 10.0.0.1 remote-as 65001

            '';

            description = ''

              Whether to enable the Configuration management daemon.

              The Configuration management daemon is automatically

              enabled if needed, at the moment this is when staticd

              is enabled.

              FRR configuration statements.

            '';

          };

          openFilesLimit = lib.mkOption {

            type = lib.types.ints.unsigned;

            default = 1024;

            description = ''

              This is the maximum number of FD's that will be available.  Use a

              reasonable value for your setup if you are expecting a large number

              of peers in say BGP.

            '';

          };

        };

      }

    { options.services.frr = (lib.genAttrs services serviceOptions); }

  ];

      { options.services.frr = (lib.genAttrs allDaemons serviceOptions); }

      (lib.mkRemovedOptionModule [ "services" "frr" "zebra" "enable" ] "FRR zebra is always enabled")

    ]

    ++ (map (d: lib.mkRenamedOptionModule [ "services" "frr" d "enable" ] [ "services" "frr" "${d}d" "enable" ]) renamedServices)

    ++ (map (d: lib.mkRenamedOptionModule [ "services" "frr" d "extraOptions" ] [ "services" "frr" "${d}d" "extraOptions" ]) (renamedServices ++ [ "static" "mgmt" ]))

    ++ (map (d: lib.mkRemovedOptionModule [ "services" "frr" d "enable" ] "FRR ${d}d is always enabled") [ "static" "mgmt" ])

    ++ (map (d: lib.mkRemovedOptionModule [ "services" "frr" d "config" ] "FRR switched to integrated-vtysh-config, please use services.frr.config") obsoleteServices)

    ++ (map (d: lib.mkRemovedOptionModule [ "services" "frr" d "configFile" ] "FRR switched to integrated-vtysh-config, please use services.frr.config or services.frr.configFile") obsoleteServices)

    ++ (map (d: lib.mkRemovedOptionModule [ "services" "frr" d "vtyListenAddress" ] "Please change -A option in services.frr.${d}.options instead") obsoleteServices)

    ++ (map (d: lib.mkRemovedOptionModule [ "services" "frr" d "vtyListenPort" ] "Please use `-P «vtyListenPort»` option with services.frr.${d}.extraOptions instead, or change services.frr.${d}.options accordingly") obsoleteServices)

    ;

  ###### implementation

  config = lib.mkIf (lib.any isEnabled allServices) {

  config =

    let

      daemonList = lib.concatStringsSep "\n" (map daemonLine daemons);

      daemonOptionLine = d: "${d}_options=\"${lib.concatStringsSep " " (cfg.${d}.options ++ cfg.${d}.extraOptions)}\"";

      daemonOptions = lib.concatStringsSep "\n" (map daemonOptionLine allDaemons);

    in

    lib.mkIf (lib.any isEnabled daemons || cfg.configFile != null || cfg.config != "") {

      environment.systemPackages = [

        pkgs.frr # for the vtysh tool

      users.groups = {

        frr = { };

        # Members of the frrvty group can use vtysh to inspect the FRR daemons

      frrvty = { members = [ "frr" ]; };

        frrvty = {

          members = [ "frr" ];

        };

    environment.etc = let

      mkEtcLink = service: {

        name = "frr/${daemonName service}.conf";

        value.source = configFile service;

      };

    in

      (builtins.listToAttrs

      (map mkEtcLink (lib.filter isEnabled allServices))) // {

        "frr/vtysh.conf".text = "";

      environment.etc = {

        "frr/frr.conf".source = configFile;

        "frr/vtysh.conf".text = ''

          service integrated-vtysh-config

        '';

        "frr/daemons".text = ''

          # This file tells the frr package which daemons to start.

          #

          # The watchfrr, zebra and staticd daemons are always started.

          #

          # This part is auto-generated from services.frr.<daemon>.enable config

          ${daemonList}

          # If this option is set the /etc/init.d/frr script automatically loads

          # the config via "vtysh -b" when the servers are started.

          #

          vtysh_enable=yes

          # This part is auto-generated from services.frr.<daemon>.options or

          # services.frr.<daemon>.extraOptions

          ${daemonOptions}

        '';

      };

    systemd.tmpfiles.rules = [

      "d /run/frr 0750 frr frr -"

    ];

      systemd.tmpfiles.rules = [ "d /run/frr 0750 frr frr -" ];

    systemd.services =

      let

        frrService = service:

          let

            scfg = cfg.${service};

            daemon = daemonName service;

          in

            lib.nameValuePair daemon ({

              wantedBy = [ "multi-user.target" ];

              after = [ "network-pre.target" "systemd-sysctl.service" ] ++ lib.optionals (service != "zebra") [ "zebra.service" ];

              bindsTo = lib.optionals (service != "zebra") [ "zebra.service" ];

      systemd.services.frr = {

        description = "FRRouting";

        documentation = [ "https://frrouting.readthedocs.io/en/latest/setup.html" ];

        wants = [ "network.target" ];

              description = if service == "zebra" then "FRR Zebra routing manager"

                else "FRR ${lib.toUpper service} routing daemon";

              unitConfig.Documentation = if service == "zebra" then "man:zebra(8)"

                else "man:${daemon}(8) man:zebra(8)";

              restartTriggers = lib.mkIf (service != "mgmt") [

                (configFile service)

        after = [

          "network-pre.target"

          "systemd-sysctl.service"

        ];

        before = [ "network.target" ];

        wantedBy = [ "multi-user.target" ];

        startLimitIntervalSec = 180;

        reloadIfChanged = true;

        restartTriggers = [

          configFile

          daemonList

        ];

              reloadIfChanged = (service != "mgmt");

        serviceConfig = {

                PIDFile = "frr/${daemon}.pid";

                ExecStart = "${pkgs.frr}/libexec/frr/${daemon}"

                  + lib.optionalString (scfg.vtyListenAddress != "") " -A ${scfg.vtyListenAddress}"

                  + lib.optionalString (scfg.vtyListenPort != null) " -P ${toString scfg.vtyListenPort}"

                  + " " + (lib.concatStringsSep " " scfg.extraOptions);

                ExecReload = lib.mkIf (service != "mgmt") "${pkgs.python3.interpreter} ${pkgs.frr}/libexec/frr/frr-reload.py --reload --daemon ${daemon} --bindir ${pkgs.frr}/bin --rundir /run/frr /etc/frr/${daemon}.conf";

                Restart = "on-abnormal";

          Nice = -5;

          Type = "forking";

          NotifyAccess = "all";

          StartLimitBurst = "3";

          TimeoutSec = 120;

          WatchdogSec = 60;

          RestartSec = 5;

          Restart = "always";

          LimitNOFILE = cfg.openFilesLimit;

          PIDFile = "/run/frr/watchfrr.pid";

          ExecStart = "${pkgs.frr}/libexec/frr/frrinit.sh start";

          ExecStop = "${pkgs.frr}/libexec/frr/frrinit.sh stop";

          ExecReload = "${pkgs.frr}/libexec/frr/frrinit.sh reload";

        };

      };

            });

       in

         lib.listToAttrs (map frrService (lib.filter isEnabled allServices));

    };

  meta.maintainers = with lib.maintainers; [ woffs ];

}

          { nodes, ... }:

          {

            virtualisation.vlans = [ 1 ];

            networking.defaultGateway = ifAddr nodes.router1 "eth1";

            services.frr = {

              config = ''

                ip route 192.168.0.0/16 ${ifAddr nodes.router1 "eth1"}

              '';

            };

          };

        router1 =

            virtualisation.vlans = [ 1 2 ];

            boot.kernel.sysctl."net.ipv4.ip_forward" = "1";

            networking.firewall.extraCommands = "iptables -A nixos-fw -i eth2 -p ospfigp -j ACCEPT";

            services.frr.ospf = {

              enable = true;

            services.frr = {

              ospfd.enable = true;

              config = ospfConf1;

            };

            specialisation.ospf.configuration = {

              services.frr.ospf.config = ospfConf2;

              services.frr.config = ospfConf2;

            };

          };

            virtualisation.vlans = [ 3 2 ];

            boot.kernel.sysctl."net.ipv4.ip_forward" = "1";

            networking.firewall.extraCommands = "iptables -A nixos-fw -i eth2 -p ospfigp -j ACCEPT";

            services.frr.ospf = {

              enable = true;

            services.frr = {

              ospfd.enable = true;

              config = ospfConf2;

            };

          };

          { nodes, ... }:

          {

            virtualisation.vlans = [ 3 ];

            networking.defaultGateway = ifAddr nodes.router2 "eth1";

            services.frr = {

              config = ''

                ip route 192.168.0.0/16 ${ifAddr nodes.router2 "eth1"}

              '';

            };

          };

      };

          for machine in client, router1, router2, server:

              machine.wait_for_unit("network.target")

          with subtest("Wait for Zebra and OSPFD"):

              for gw in router1, router2:

                  gw.wait_for_unit("zebra")

                  gw.wait_for_unit("ospfd")

          with subtest("Wait for FRR"):

              for gw in client, router1, router2, server:

                  gw.wait_for_unit("frr")

          router1.succeed("${nodes.router1.config.system.build.toplevel}/specialisation/ospf/bin/switch-to-configuration test >&2")

    "--enable-user=frr"

    "--enable-vty-group=frrvty"

    "--localstatedir=/run/frr"

    "--sbindir=$(out)/libexec/frr"

    "--sbindir=${placeholder "out"}/libexec/frr"

    "--sysconfdir=/etc/frr"

    "--with-clippy=${finalAttrs.clippy-helper}/bin/clippy"

    # general options

  postPatch = ''

    substituteInPlace tools/frr-reload \

      --replace /usr/lib/frr/ $out/libexec/frr/

      --replace-quiet /usr/lib/frr/ $out/libexec/frr/

    sed -i '/^PATH=/ d' tools/frr.in tools/frrcommon.sh.in

  '';

  doCheck = true;