NixOS-Config/hosts/firefly/boot/default.nix

{ pkgs, lib, ... }:
let
  commonKernelParams = [
    # Nvidia settings
    "nvidia_drm.fbdev=1"
    "nouveau.config=NvGspRm=1"

    # VM/GPU passthrough
    "amd_iommu=on"
    "iommu=pt"
    "nested=1"

    # Virtualization nonsense
    "transparent_hugepage=never"

    # Isolate devices into IOMMU groups
    "pcie_acs_override=downstream,multifunction"
    "pci=routeirq"
  ];
in {
  boot = {
    # Must be unstable for newest NVIDIA drivers
    kernelPackages = pkgs.unstable.linuxPackages_latest;
    blacklistedKernelModules = [ "pcspkr" ];
    kernel.sysctl."vm.max_map_count" = 2147483642;
    kernelParams = commonKernelParams ++ [ "vfio-pci.ids=10de:1f82,10de:10fa" ];

    initrd = {
      systemd = {
        enable = true;
        services.root-reset = {
          description = "Reset root and snapshot last boot";
          wantedBy = [ "initrd.target" ];
          after = [ "dev-nixos-root.device" ];
          before = [ "sysroot.mount" ];
          unitConfig.DefaultDependencies = "no";
          serviceConfig.Type = "oneshot";
          script = ''
            mkdir -p /mnt
            mount /dev/nixos/root /mnt

            if [[ -e /mnt/prev ]]; then
              btrfs subvolume delete /mnt/prev
            fi

            btrfs subvolume snapshot /mnt/root /mnt/prev

            btrfs subvolume list -o /mnt/root | cut -f9 -d' ' | while read subvolume; do
              btrfs subvolume delete "/mnt/$subvolume"
            done

            btrfs subvolume delete /mnt/root
            btrfs subvolume create /mnt/root

            umount /mnt
          '';
	};
      };
    };
  };

  # Additional entry to boot from the second GPU
  specialisation = {
    gputwo.configuration = {
      boot.kernelParams = commonKernelParams ++ [ "vfio-pci.ids=10de:2504,10de:228e" ];
    };
  };
}