{ ... }: { flake.nixosModules.systemGpuPassthru = { config, lib, pkgs, ... }: let cfg = config.chiasson.system.gpuPassthru; vfioIds = lib.concatStringsSep "," cfg.pciIds; # Must match the Windows host installer build (B7-244-4bb2c58f). lgRev = "4bb2c58fb6d0df9e863ad45924dd4decc7e9cf4e"; lgVersion = "B7-244-4bb2c58f"; # Tarball hashes omit submodules; fetchSubmodules needs the git tree hash. lgHash = "sha256-Dc0sFnJVM0xJ0FfIQqGQKxFUj6mhp3qgB772h3Dgowc="; # https://wiki.nixos.org/wiki/Looking_Glass — systemd 258+ needs namespaces=[] # and an explicit ACL; seccomp can block mmap on /dev/kvmfr0. lgQemuVerbatimConfig = '' namespaces = [] seccomp_sandbox = 0 security_driver = "none" security_default_confined = 0 security_require_confined = 0 cgroup_device_acl = [ "/dev/null", "/dev/full", "/dev/zero", "/dev/random", "/dev/urandom", "/dev/ptmx", "/dev/kvm", "/dev/kqemu", "/dev/rtc", "/dev/hpet", "/dev/vfio/", "/dev/vfio/vfio", "/dev/kvmfr0", "c 234:* rwm", "/dev/disk/", "/dev/disk/by-id/", "/dev/bus/usb/", ] ''; in { options.chiasson.system.gpuPassthru = { enable = lib.mkEnableOption '' Bind passthrough GPU(s) to VFIO, run the host on Intel iGPU, and enable libvirt + Looking Glass for a Windows gaming VM. Prerequisites (manual, one-time): - BIOS: VT-d/IOMMU on, Above 4G Decoding / Resizable BAR on - Move host monitors to motherboard (Intel) ports; dummy HDMI/DP plug on passthrough GPU - Wipe the VM disk before first boot: `wipefs -a ` - Verify IOMMU groups after switch: GPU functions should be isolated Rollout: leave `enable = false` until cabling and disk backup are done, then rebuild and switch. ''; pciIds = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; example = [ "10de:1f07" "10de:10f9" "10de:1ada" "10de:1adb" ]; description = '' PCI vendor:device IDs to bind to vfio-pci (all GPU functions: VGA, audio, USB, etc.). ''; }; windowsDisk = lib.mkOption { type = lib.types.str; default = ""; example = "/dev/disk/by-id/ata-CT1000MX500SSD1_1830E14B054A"; description = '' Whole-disk by-id path for the Windows VM (raw block passthrough). Must not be mounted by the host. ''; }; lookingGlass = { enable = lib.mkOption { type = lib.types.bool; default = true; description = '' Load the kvmfr kernel module and grant QEMU access to /dev/kvmfr0 for Looking Glass (required for NVIDIA VFIO; plain /dev/shm IVSHMEM causes DMA mapping failures). ''; }; shmSizeMiB = lib.mkOption { type = lib.types.int; default = 32; description = '' kvmfr shared memory size (MiB). Must match `kvmfr.static_size_mb` and the VM `memory-backend-file` size in the domain XML. ''; }; captureOnFocus = lib.mkOption { type = lib.types.bool; default = true; description = '' Grab keyboard/mouse when the Looking Glass window is focused. Set `false` to keep input on the Linux host while viewing the guest. ''; }; }; vm = { enable = lib.mkOption { type = lib.types.bool; default = false; description = '' Declaratively define the Windows VM via NixVirt (host module must import NixVirt and set `virtualisation.libvirt.connections`). ''; }; name = lib.mkOption { type = lib.types.str; default = "win11"; description = "libvirt domain name."; }; memoryGiB = lib.mkOption { type = lib.types.int; default = 16; description = "VM RAM in GiB (informational; domain XML is authoritative)."; }; vcpus = lib.mkOption { type = lib.types.int; default = 8; description = "vCPU count (informational; domain XML is authoritative)."; }; installMode = lib.mkOption { type = lib.types.bool; default = true; description = '' Use install domain (ISO, no GPU). Set `false` after Windows is installed. ''; }; }; hugepages.enable = lib.mkOption { type = lib.types.bool; default = false; description = "Reserve 2 MiB hugepages for Looking Glass IVSHMEM (enable if frame drops)."; }; libvirtUsers = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; description = "Users added to libvirtd, kvm, and disk groups."; }; }; config = lib.mkIf cfg.enable (lib.mkMerge [ { assertions = [ { assertion = cfg.pciIds != [ ]; message = "chiasson.system.gpuPassthru: pciIds must list every passthrough GPU function."; } { assertion = cfg.windowsDisk != ""; message = "chiasson.system.gpuPassthru: windowsDisk must be set to a /dev/disk/by-id/ path."; } { assertion = cfg.libvirtUsers != [ ]; message = "chiasson.system.gpuPassthru: libvirtUsers must name at least one user."; } ]; boot.kernelParams = [ "intel_iommu=on" "iommu=pt" "kvm.ignore_msrs=1" "vfio-pci.ids=${vfioIds}" ]; boot.initrd.kernelModules = [ "vfio_pci" "vfio" "vfio_iommu_type1" ]; boot.blacklistedKernelModules = [ "nouveau" "nvidia" "nvidia_drm" "nvidia_modeset" "i2c_nvidia_gpu" "mt7921e" "mt7925e" ]; boot.extraModprobeConfig = '' softdep nvidia pre: vfio-pci softdep nvidia_drm pre: vfio-pci softdep nouveau pre: vfio-pci options vfio_iommu_type1 allow_unsafe_interrupts=1 ''; services.xserver.videoDrivers = lib.mkForce [ "modesetting" ]; hardware.nvidia-container-toolkit.enable = lib.mkForce false; chiasson.system.gaming.sunshine.cudaSupport = lib.mkForce false; # libvirt enables ZFS storage pools on Linux by default, which pulls in # zfs-user (currently broken to build on nixos-unstable). Raw block disk # passthrough does not need libvirt ZFS support. nixpkgs.overlays = [ (final: prev: { libvirt = prev.libvirt.override { enableZfs = false; }; }) ] ++ lib.optionals cfg.lookingGlass.enable [ (final: prev: { looking-glass-client = prev.looking-glass-client.overrideAttrs (old: { version = lgVersion; src = prev.fetchFromGitHub { owner = "gnif"; repo = "LookingGlass"; rev = lgRev; hash = lgHash; fetchSubmodules = true; }; patches = old.patches; postUnpack = '' echo ${lgRev} > $sourceRoot/VERSION export sourceRoot="$sourceRoot/client" ''; }); }) ]; programs.virt-manager.enable = true; virtualisation.spiceUSBRedirection.enable = true; virtualisation.libvirtd = { enable = true; onBoot = "ignore"; onShutdown = "shutdown"; qemu = { package = pkgs.qemu_kvm; # GPU + raw block passthrough needs root (or fragile per-device ACLs). runAsRoot = true; swtpm.enable = true; verbatimConfig = lib.mkIf cfg.lookingGlass.enable (lib.mkForce lgQemuVerbatimConfig); }; }; systemd.services.libvirtd.restartTriggers = [ "${pkgs.qemu_kvm}/bin/qemu-system-x86_64" ]; environment.systemPackages = with pkgs; [ virt-manager qemu_kvm OVMF edk2 looking-glass-client swtpm ] ++ lib.optionals cfg.lookingGlass.enable [ (writeShellScriptBin "looking-glass-toggle" '' set -euo pipefail pkill -f 'looking-glass-client' || true exec looking-glass-client -F -m 97 '') ]; # B7 defaults to /dev/kvmfr0; pin SPICE to the fixed port in the domain XMLs. environment.etc."looking-glass-client.ini" = lib.mkIf cfg.lookingGlass.enable (lib.mkForce { text = '' [app] shmFile=/dev/kvmfr0 allowDMA=yes [win] fullScreen=yes [spice] enable=yes host=127.0.0.1 port=5900 input=yes audio=yes clipboard=yes [input] captureOnFocus=${lib.boolToString cfg.lookingGlass.captureOnFocus} escapeKey=97 ''; }); users.groups.libvirtd.members = cfg.libvirtUsers; users.users = lib.genAttrs cfg.libvirtUsers (_: { extraGroups = [ "libvirtd" "kvm" "disk" ]; }) // { qemu-libvirtd.extraGroups = [ "kvm" "disk" ]; }; } { systemd.tmpfiles.rules = [ "d /var/lib/libvirt/boot 0755 root root -" ]; } (lib.mkIf cfg.lookingGlass.enable { boot.extraModulePackages = lib.mkAfter [ config.boot.kernelPackages.kvmfr ]; boot.initrd.kernelModules = lib.mkAfter [ "kvmfr" ]; boot.kernelModules = lib.mkAfter [ "kvmfr" ]; boot.kernelParams = lib.mkAfter [ "kvmfr.static_size_mb=${toString cfg.lookingGlass.shmSizeMiB}" ]; boot.extraModprobeConfig = lib.mkAfter '' options kvmfr static_size_mb=${toString cfg.lookingGlass.shmSizeMiB} ''; services.udev.packages = [ (pkgs.writeTextFile { name = "kvmfr-udev"; text = '' SUBSYSTEM=="kvmfr", GROUP="kvm", MODE="0660", TAG+="uaccess" ''; destination = "/etc/udev/rules.d/70-kvmfr.rules"; }) ]; # insmod from the nix store: extraModulePackages only lands in # /run/booted-system/kernel-modules after a reboot. systemd.services.kvmfr = { description = "Looking Glass kvmfr module"; wantedBy = [ "multi-user.target" ]; before = [ "libvirtd.service" "libvirtd-config.service" "libvirt-guests.service" ]; after = [ "systemd-modules-load.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = pkgs.writeShellScript "kvmfr-start" ( let kvmfrKo = "${config.boot.kernelPackages.kvmfr}/lib/modules/${config.boot.kernelPackages.kernel.modDirVersion}/kernel/drivers/misc/kvmfr.ko"; shmMiB = toString cfg.lookingGlass.shmSizeMiB; in '' set -euo pipefail if [ -e /dev/kvmfr0 ] && [ ! -c /dev/kvmfr0 ]; then rm -f /dev/kvmfr0 fi if [ -c /dev/kvmfr0 ]; then exit 0 fi if ${pkgs.kmod}/bin/lsmod | ${pkgs.gnugrep}/bin/grep -q '^kvmfr '; then ${pkgs.kmod}/bin/rmmod kvmfr fi if [ ! -f ${kvmfrKo} ]; then echo "kvmfr: module missing at ${kvmfrKo}" >&2 exit 1 fi ${pkgs.kmod}/bin/insmod ${kvmfrKo} static_size_mb=${shmMiB} if [ ! -c /dev/kvmfr0 ]; then echo "kvmfr: /dev/kvmfr0 not created (static_size_mb=${shmMiB})" >&2 exit 1 fi '' ); }; }; systemd.services.libvirtd = lib.mkIf cfg.lookingGlass.enable { after = [ "kvmfr.service" ]; requires = [ "kvmfr.service" ]; }; }) (lib.mkIf cfg.hugepages.enable { boot.kernelParams = [ "hugepagesz=2M" "hugepages=64" ]; }) ]); }; }