NixOS-Config/hosts/tower/boot/default.nix

{ config, pkgs, ... }:
let
  commonKernelParams = [
    # VM/GPU passthrough
    "amd_iommu=on"
    "iommu=pt"
    "nested=1"

    # Virtualization nonsense
    "transparent_hugepage=never"

    # Isolate devices into IOMMU groups
    "pcie_acs_override=downstream,multifunction"
    "pci=routeirq"
  ];
in {
  boot = {
    # Must be Zen for IOMMU isolation
    kernelPackages = pkgs.unstable.linuxPackages_zen;
    kernel.sysctl."vm.max_map_count" = 2147483642;
    kernelParams = commonKernelParams ++ [ "vfio-pci.ids=10de:1f82,10de:10fa" ];
    blacklistedKernelModules = [ "pcspkr" ];

    # Needed for GPU passthrough
    initrd.kernelModules = [
      "vfio"
      "vfio_pci"
      "vfio_iommu_type1"
    ];

    initrd.systemd = {
      enable = true;
      services.root-reset = {
        description = "Reset root and snapshot last boot";
        wantedBy = [ "initrd.target" ];
        before = [ "sysroot.mount" ];
        after = [ "dev-${config.networking.hostName}-root.device" ];
        unitConfig.DefaultDependencies = "no";
        serviceConfig.Type = "oneshot";
        script = ''
          mkdir -p /mnt
          mount /dev/${config.networking.hostName}/root /mnt

          if [[ -e /mnt/prev ]]; then
            btrfs subvolume delete /mnt/prev
          fi

          btrfs subvolume snapshot /mnt/root /mnt/prev

          btrfs subvolume list -o /mnt/root | cut -f9 -d' ' | while read subvolume; do
            btrfs subvolume delete "/mnt/$subvolume"
          done

          btrfs subvolume delete /mnt/root
          btrfs subvolume create /mnt/root

          umount /mnt
        '';
      };
    };
  };

  # Use second GPU on boot
  specialisation.gputwo.configuration = {
    boot.kernelParams = commonKernelParams ++ [ "vfio-pci.ids=10de:2504,10de:228e" ];
  };
}