Merge pull request #201676 from DeterminateSystems/ec2-metadata-fetch-nofail

  </section>

  <section xml:id="sec-release-23.05-incompatibilities">

    <title>Backward Incompatibilities</title>

    <itemizedlist spacing="compact">

    <itemizedlist>

      <listitem>

        <para>

          <literal>carnix</literal> and <literal>cratesIO</literal> has

          instead.

        </para>

      </listitem>

      <listitem>

        <para>

          The EC2 image module no longer fetches instance metadata in

          stage-1. This results in a significantly smaller initramfs,

          since network drivers no longer need to be included, and

          faster boots, since metadata fetching can happen in parallel

          with startup of other services. This breaks services which

          rely on metadata being present by the time stage-2 is entered.

          Anything which reads EC2 metadata from

          <literal>/etc/ec2-metadata</literal> should now have an

          <literal>after</literal> dependency on

          <literal>fetch-ec2-metadata.service</literal>

        </para>

      </listitem>

      <listitem>

        <para>

          The EC2 image module previously detected and automatically

          mounted ext3-formatted instance store devices and partitions

          in stage-1 (initramfs), storing <literal>/tmp</literal> on the

          first discovered device. This behaviour, which only catered to

          very specific use cases and could not be disabled, has been

          removed. Users relying on this should provide their own

          implementation, and probably use ext4 and perform the mount in

          stage-2.

        </para>

      </listitem>

      <listitem>

        <para>

          The EC2 image module previously detected and activated

          swap-formatted instance store devices and partitions in

          stage-1 (initramfs). This behaviour has been removed. Users

          relying on this should provide their own implementation.

        </para>

      </listitem>

    </itemizedlist>

  </section>

  <section xml:id="sec-release-23.05-notable-changes">

- `carnix` and `cratesIO` has been removed due to being unmaintained, use alternatives such as [naersk](https://github.com/nix-community/naersk) and [crate2nix](https://github.com/kolloch/crate2nix) instead.

- The EC2 image module no longer fetches instance metadata in stage-1. This results in a significantly smaller initramfs, since network drivers no longer need to be included, and faster boots, since metadata fetching can happen in parallel with startup of other services.

  This breaks services which rely on metadata being present by the time stage-2 is entered. Anything which reads EC2 metadata from `/etc/ec2-metadata` should now have an `after` dependency on `fetch-ec2-metadata.service`

- The EC2 image module previously detected and automatically mounted ext3-formatted instance store devices and partitions in stage-1 (initramfs), storing `/tmp` on the first discovered device. This behaviour, which only catered to very specific use cases and could not be disabled, has been removed. Users relying on this should provide their own implementation, and probably use ext4 and perform the mount in stage-2.

- The EC2 image module previously detected and activated swap-formatted instance store devices and partitions in stage-1 (initramfs). This behaviour has been removed. Users relying on this should provide their own implementation.

## Other Notable Changes {#sec-release-23.05-notable-changes}

<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

    sizeMB = mkOption {

      type = with types; either (enum [ "auto" ]) int;

      default = if config.ec2.hvm then 2048 else 8192;

      default = 2048;

      example = 8192;

      description = lib.mdDoc "The size in MB of the image";

    };

      ''

        { modulesPath, ... }: {

          imports = [ "''${modulesPath}/virtualisation/amazon-image.nix" ];

          ${optionalString config.ec2.hvm ''

            ec2.hvm = true;

          ''}

          ${optionalString config.ec2.efi ''

            ec2.efi = true;

          ''}

      pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package

      fsType = "ext4";

      partitionTableType = if config.ec2.efi then "efi"

                           else if config.ec2.hvm then "legacy+gpt"

                           else "none";

      partitionTableType = if config.ec2.efi then "efi" else "legacy+gpt";

      diskSize = cfg.sizeMB;

let

  cfg = config.ec2;

  metadataFetcher = import ./ec2-metadata-fetcher.nix {

    inherit (pkgs) curl;

    targetRoot = "$targetRoot/";

    wgetExtraOptions = "-q";

  };

in

{

  config = {

    assertions = [

      { assertion = cfg.hvm;

        message = "Paravirtualized EC2 instances are no longer supported.";

      }

      { assertion = cfg.efi -> cfg.hvm;

        message = "EC2 instances using EFI must be HVM instances.";

      }

      { assertion = versionOlder config.boot.kernelPackages.kernel.version "5.17";

        message = "ENA driver fails to build with kernel >= 5.17";

      }

    ];

    boot.growPartition = cfg.hvm;

    boot.growPartition = true;

    fileSystems."/" = mkIf (!cfg.zfs.enable) {

      device = "/dev/disk/by-label/nixos";

    boot.extraModulePackages = [

      config.boot.kernelPackages.ena

    ];

    boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ];

    boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ];

    boot.kernelParams = mkIf cfg.hvm [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];

    boot.initrd.kernelModules = [ "xen-blkfront" ];

    boot.initrd.availableKernelModules = [ "nvme" ];

    boot.kernelParams = [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];

    # Prevent the nouveau kernel module from being loaded, as it

    # interferes with the nvidia/nvidia-uvm modules needed for CUDA.

    # boot.

    boot.blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];

    # Generate a GRUB menu.  Amazon's pv-grub uses this to boot our kernel/initrd.

    boot.loader.grub.version = if cfg.hvm then 2 else 1;

    boot.loader.grub.device = if (cfg.hvm && !cfg.efi) then "/dev/xvda" else "nodev";

    boot.loader.grub.extraPerEntryConfig = mkIf (!cfg.hvm) "root (hd0)";

    boot.loader.grub.device = if cfg.efi then "nodev" else "/dev/xvda";

    boot.loader.grub.efiSupport = cfg.efi;

    boot.loader.grub.efiInstallAsRemovable = cfg.efi;

    boot.loader.timeout = 1;

      terminal_input console serial

    '';

    boot.initrd.network.enable = true;

    # Mount all formatted ephemeral disks and activate all swap devices.

    # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options

    # because the set of devices is dependent on the instance type

    # (e.g. "m1.small" has one ephemeral filesystem and one swap device,

    # while "m1.large" has two ephemeral filesystems and no swap

    # devices).  Also, put /tmp and /var on /disk0, since it has a lot

    # more space than the root device.  Similarly, "move" /nix to /disk0

    # by layering a unionfs-fuse mount on top of it so we have a lot more space for

    # Nix operations.

    boot.initrd.postMountCommands =

      ''

        ${metadataFetcher}

        diskNr=0

        diskForUnionfs=

        for device in /dev/xvd[abcde]*; do

            if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi

            fsType=$(blkid -o value -s TYPE "$device" || true)

            if [ "$fsType" = swap ]; then

                echo "activating swap device $device..."

                swapon "$device" || true

            elif [ "$fsType" = ext3 ]; then

                mp="/disk$diskNr"

                diskNr=$((diskNr + 1))

                if mountFS "$device" "$mp" "" ext3; then

                    if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi

                fi

            else

                echo "skipping unknown device type $device"

            fi

        done

        if [ -n "$diskForUnionfs" ]; then

            mkdir -m 755 -p $targetRoot/$diskForUnionfs/root

            mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp

            mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp

            if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then

                mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var

                mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var

                mkdir -p /unionfs-chroot/ro-nix

                mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix

                mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix

                mkdir -p /unionfs-chroot/rw-nix

                mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix

                unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix

            fi

        fi

      '';

    boot.initrd.extraUtilsCommands =

      ''

        # We need swapon in the initrd.

        copy_bin_and_libs ${pkgs.util-linux}/sbin/swapon

      '';

    systemd.services.fetch-ec2-metadata = {

      wantedBy = [ "multi-user.target" ];

      after = ["network-online.target"];

      path = [ pkgs.curl ];

      script = builtins.readFile ./ec2-metadata-fetcher.sh;

      serviceConfig.Type = "oneshot";

      serviceConfig.StandardOutput = "journal+console";

    };

    # Allow root logins only using the SSH key that the user specified

    # at instance creation time.

    # Always include cryptsetup so that Charon can use it.

    environment.systemPackages = [ pkgs.cryptsetup ];

    boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];

    # EC2 has its own NTP server provided by the hypervisor

    networking.timeServers = [ "169.254.169.123" ];

let

  inherit (lib) literalExpression types;

in {

  imports = [

    (lib.mkRemovedOptionModule [ "ec2" "hvm" ] "Only HVM instances are supported, so specifying it is no longer necessary.")

  ];

  options = {

    ec2 = {

      zfs = {

          });

        };

      };

      hvm = lib.mkOption {

        default = lib.versionAtLeast config.system.stateVersion "17.03";

        internal = true;

        description = lib.mdDoc ''

          Whether the EC2 instance is a HVM instance.

        '';

      };

      efi = lib.mkOption {

        default = pkgs.stdenv.hostPlatform.isAarch64;

        defaultText = literalExpression "pkgs.stdenv.hostPlatform.isAarch64";