384 lines
13 KiB
Nix
384 lines
13 KiB
Nix
{ ... }: {
|
|
flake.nixosModules.systemGpuPassthru =
|
|
{ config, lib, pkgs, ... }:
|
|
let
|
|
cfg = config.chiasson.system.gpuPassthru;
|
|
vfioIds = lib.concatStringsSep "," cfg.pciIds;
|
|
# Must match the Windows host installer build (B7-244-4bb2c58f).
|
|
lgRev = "4bb2c58fb6d0df9e863ad45924dd4decc7e9cf4e";
|
|
lgVersion = "B7-244-4bb2c58f";
|
|
# Tarball hashes omit submodules; fetchSubmodules needs the git tree hash.
|
|
lgHash = "sha256-Dc0sFnJVM0xJ0FfIQqGQKxFUj6mhp3qgB772h3Dgowc=";
|
|
# https://wiki.nixos.org/wiki/Looking_Glass — systemd 258+ needs namespaces=[]
|
|
# and an explicit ACL; seccomp can block mmap on /dev/kvmfr0.
|
|
lgQemuVerbatimConfig = ''
|
|
namespaces = []
|
|
seccomp_sandbox = 0
|
|
security_driver = "none"
|
|
security_default_confined = 0
|
|
security_require_confined = 0
|
|
cgroup_device_acl = [
|
|
"/dev/null", "/dev/full", "/dev/zero",
|
|
"/dev/random", "/dev/urandom",
|
|
"/dev/ptmx", "/dev/kvm", "/dev/kqemu",
|
|
"/dev/rtc", "/dev/hpet",
|
|
"/dev/vfio/", "/dev/vfio/vfio",
|
|
"/dev/kvmfr0", "c 234:* rwm",
|
|
"/dev/disk/", "/dev/disk/by-id/",
|
|
"/dev/bus/usb/",
|
|
]
|
|
'';
|
|
in
|
|
{
|
|
options.chiasson.system.gpuPassthru = {
|
|
enable = lib.mkEnableOption ''
|
|
Bind passthrough GPU(s) to VFIO, run the host on Intel iGPU, and enable libvirt +
|
|
Looking Glass for a Windows gaming VM.
|
|
|
|
Prerequisites (manual, one-time):
|
|
- BIOS: VT-d/IOMMU on, Above 4G Decoding / Resizable BAR on
|
|
- Move host monitors to motherboard (Intel) ports; dummy HDMI/DP plug on passthrough GPU
|
|
- Wipe the VM disk before first boot: `wipefs -a <windowsDisk>`
|
|
- Verify IOMMU groups after switch: GPU functions should be isolated
|
|
|
|
Rollout: leave `enable = false` until cabling and disk backup are done, then rebuild and switch.
|
|
'';
|
|
|
|
pciIds = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
example = [ "10de:1f07" "10de:10f9" "10de:1ada" "10de:1adb" ];
|
|
description = ''
|
|
PCI vendor:device IDs to bind to vfio-pci (all GPU functions: VGA, audio, USB, etc.).
|
|
'';
|
|
};
|
|
|
|
windowsDisk = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "";
|
|
example = "/dev/disk/by-id/ata-CT1000MX500SSD1_1830E14B054A";
|
|
description = ''
|
|
Whole-disk by-id path for the Windows VM (raw block passthrough). Must not be mounted by the host.
|
|
'';
|
|
};
|
|
|
|
lookingGlass = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
description = ''
|
|
Load the kvmfr kernel module and grant QEMU access to /dev/kvmfr0 for Looking Glass
|
|
(required for NVIDIA VFIO; plain /dev/shm IVSHMEM causes DMA mapping failures).
|
|
'';
|
|
};
|
|
|
|
shmSizeMiB = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 32;
|
|
description = ''
|
|
kvmfr shared memory size (MiB). Must match `kvmfr.static_size_mb` and the VM
|
|
`memory-backend-file` size in the domain XML.
|
|
'';
|
|
};
|
|
|
|
captureOnFocus = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
description = ''
|
|
Grab keyboard/mouse when the Looking Glass window is focused. Set `false` to
|
|
keep input on the Linux host while viewing the guest.
|
|
'';
|
|
};
|
|
};
|
|
|
|
vm = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = ''
|
|
Declaratively define the Windows VM via NixVirt (host module must import NixVirt and set
|
|
`virtualisation.libvirt.connections`).
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "win11";
|
|
description = "libvirt domain name.";
|
|
};
|
|
|
|
memoryGiB = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 16;
|
|
description = "VM RAM in GiB (informational; domain XML is authoritative).";
|
|
};
|
|
|
|
vcpus = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 8;
|
|
description = "vCPU count (informational; domain XML is authoritative).";
|
|
};
|
|
|
|
installMode = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
description = ''
|
|
Use install domain (ISO, no GPU). Set `false` after Windows is installed.
|
|
'';
|
|
};
|
|
};
|
|
|
|
hugepages.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = "Reserve 2 MiB hugepages for Looking Glass IVSHMEM (enable if frame drops).";
|
|
};
|
|
|
|
libvirtUsers = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
description = "Users added to libvirtd, kvm, and disk groups.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable (lib.mkMerge [
|
|
{
|
|
assertions = [
|
|
{
|
|
assertion = cfg.pciIds != [ ];
|
|
message = "chiasson.system.gpuPassthru: pciIds must list every passthrough GPU function.";
|
|
}
|
|
{
|
|
assertion = cfg.windowsDisk != "";
|
|
message = "chiasson.system.gpuPassthru: windowsDisk must be set to a /dev/disk/by-id/ path.";
|
|
}
|
|
{
|
|
assertion = cfg.libvirtUsers != [ ];
|
|
message = "chiasson.system.gpuPassthru: libvirtUsers must name at least one user.";
|
|
}
|
|
];
|
|
|
|
boot.kernelParams = [
|
|
"intel_iommu=on"
|
|
"iommu=pt"
|
|
"kvm.ignore_msrs=1"
|
|
"vfio-pci.ids=${vfioIds}"
|
|
];
|
|
|
|
boot.initrd.kernelModules = [
|
|
"vfio_pci"
|
|
"vfio"
|
|
"vfio_iommu_type1"
|
|
];
|
|
|
|
boot.blacklistedKernelModules = [
|
|
"nouveau"
|
|
"nvidia"
|
|
"nvidia_drm"
|
|
"nvidia_modeset"
|
|
"i2c_nvidia_gpu"
|
|
"mt7921e"
|
|
"mt7925e"
|
|
];
|
|
|
|
boot.extraModprobeConfig = ''
|
|
softdep nvidia pre: vfio-pci
|
|
softdep nvidia_drm pre: vfio-pci
|
|
softdep nouveau pre: vfio-pci
|
|
options vfio_iommu_type1 allow_unsafe_interrupts=1
|
|
'';
|
|
|
|
services.xserver.videoDrivers = lib.mkForce [ "modesetting" ];
|
|
|
|
hardware.nvidia-container-toolkit.enable = lib.mkForce false;
|
|
chiasson.system.gaming.sunshine.cudaSupport = lib.mkForce false;
|
|
|
|
# libvirt enables ZFS storage pools on Linux by default, which pulls in
|
|
# zfs-user (currently broken to build on nixos-unstable). Raw block disk
|
|
# passthrough does not need libvirt ZFS support.
|
|
nixpkgs.overlays =
|
|
[
|
|
(final: prev: {
|
|
libvirt = prev.libvirt.override { enableZfs = false; };
|
|
})
|
|
]
|
|
++ lib.optionals cfg.lookingGlass.enable [
|
|
(final: prev: {
|
|
looking-glass-client = prev.looking-glass-client.overrideAttrs (old: {
|
|
version = lgVersion;
|
|
src = prev.fetchFromGitHub {
|
|
owner = "gnif";
|
|
repo = "LookingGlass";
|
|
rev = lgRev;
|
|
hash = lgHash;
|
|
fetchSubmodules = true;
|
|
};
|
|
patches = old.patches;
|
|
postUnpack = ''
|
|
echo ${lgRev} > $sourceRoot/VERSION
|
|
export sourceRoot="$sourceRoot/client"
|
|
'';
|
|
});
|
|
})
|
|
];
|
|
|
|
programs.virt-manager.enable = true;
|
|
virtualisation.spiceUSBRedirection.enable = true;
|
|
|
|
virtualisation.libvirtd = {
|
|
enable = true;
|
|
onBoot = "ignore";
|
|
onShutdown = "shutdown";
|
|
qemu = {
|
|
package = pkgs.qemu_kvm;
|
|
# GPU + raw block passthrough needs root (or fragile per-device ACLs).
|
|
runAsRoot = true;
|
|
swtpm.enable = true;
|
|
verbatimConfig = lib.mkIf cfg.lookingGlass.enable (lib.mkForce lgQemuVerbatimConfig);
|
|
};
|
|
};
|
|
|
|
systemd.services.libvirtd.restartTriggers = [
|
|
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
|
|
];
|
|
|
|
environment.systemPackages =
|
|
with pkgs;
|
|
[
|
|
virt-manager
|
|
qemu_kvm
|
|
OVMF
|
|
edk2
|
|
looking-glass-client
|
|
swtpm
|
|
]
|
|
++ lib.optionals cfg.lookingGlass.enable [
|
|
(writeShellScriptBin "looking-glass-toggle" ''
|
|
set -euo pipefail
|
|
pkill -f 'looking-glass-client' || true
|
|
exec looking-glass-client -F -m 97
|
|
'')
|
|
];
|
|
|
|
# B7 defaults to /dev/kvmfr0; pin SPICE to the fixed port in the domain XMLs.
|
|
environment.etc."looking-glass-client.ini" = lib.mkIf cfg.lookingGlass.enable (lib.mkForce {
|
|
text = ''
|
|
[app]
|
|
shmFile=/dev/kvmfr0
|
|
allowDMA=yes
|
|
|
|
[win]
|
|
fullScreen=yes
|
|
|
|
[spice]
|
|
enable=yes
|
|
host=127.0.0.1
|
|
port=5900
|
|
input=yes
|
|
audio=yes
|
|
clipboard=yes
|
|
|
|
[input]
|
|
captureOnFocus=${lib.boolToString cfg.lookingGlass.captureOnFocus}
|
|
escapeKey=97
|
|
'';
|
|
});
|
|
|
|
users.groups.libvirtd.members = cfg.libvirtUsers;
|
|
users.users =
|
|
lib.genAttrs cfg.libvirtUsers (_: {
|
|
extraGroups = [
|
|
"libvirtd"
|
|
"kvm"
|
|
"disk"
|
|
];
|
|
})
|
|
// {
|
|
qemu-libvirtd.extraGroups = [
|
|
"kvm"
|
|
"disk"
|
|
];
|
|
};
|
|
}
|
|
{
|
|
systemd.tmpfiles.rules = [
|
|
"d /var/lib/libvirt/boot 0755 root root -"
|
|
];
|
|
}
|
|
(lib.mkIf cfg.lookingGlass.enable {
|
|
boot.extraModulePackages = lib.mkAfter [ config.boot.kernelPackages.kvmfr ];
|
|
boot.initrd.kernelModules = lib.mkAfter [ "kvmfr" ];
|
|
boot.kernelModules = lib.mkAfter [ "kvmfr" ];
|
|
boot.kernelParams = lib.mkAfter [
|
|
"kvmfr.static_size_mb=${toString cfg.lookingGlass.shmSizeMiB}"
|
|
];
|
|
|
|
boot.extraModprobeConfig = lib.mkAfter ''
|
|
options kvmfr static_size_mb=${toString cfg.lookingGlass.shmSizeMiB}
|
|
'';
|
|
|
|
services.udev.packages = [
|
|
(pkgs.writeTextFile {
|
|
name = "kvmfr-udev";
|
|
text = ''
|
|
SUBSYSTEM=="kvmfr", GROUP="kvm", MODE="0660", TAG+="uaccess"
|
|
'';
|
|
destination = "/etc/udev/rules.d/70-kvmfr.rules";
|
|
})
|
|
];
|
|
|
|
# insmod from the nix store: extraModulePackages only lands in
|
|
# /run/booted-system/kernel-modules after a reboot.
|
|
systemd.services.kvmfr = {
|
|
description = "Looking Glass kvmfr module";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [
|
|
"libvirtd.service"
|
|
"libvirtd-config.service"
|
|
"libvirt-guests.service"
|
|
];
|
|
after = [ "systemd-modules-load.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = pkgs.writeShellScript "kvmfr-start" (
|
|
let
|
|
kvmfrKo = "${config.boot.kernelPackages.kvmfr}/lib/modules/${config.boot.kernelPackages.kernel.modDirVersion}/kernel/drivers/misc/kvmfr.ko";
|
|
shmMiB = toString cfg.lookingGlass.shmSizeMiB;
|
|
in
|
|
''
|
|
set -euo pipefail
|
|
if [ -e /dev/kvmfr0 ] && [ ! -c /dev/kvmfr0 ]; then
|
|
rm -f /dev/kvmfr0
|
|
fi
|
|
if [ -c /dev/kvmfr0 ]; then
|
|
exit 0
|
|
fi
|
|
if ${pkgs.kmod}/bin/lsmod | ${pkgs.gnugrep}/bin/grep -q '^kvmfr '; then
|
|
${pkgs.kmod}/bin/rmmod kvmfr
|
|
fi
|
|
if [ ! -f ${kvmfrKo} ]; then
|
|
echo "kvmfr: module missing at ${kvmfrKo}" >&2
|
|
exit 1
|
|
fi
|
|
${pkgs.kmod}/bin/insmod ${kvmfrKo} static_size_mb=${shmMiB}
|
|
if [ ! -c /dev/kvmfr0 ]; then
|
|
echo "kvmfr: /dev/kvmfr0 not created (static_size_mb=${shmMiB})" >&2
|
|
exit 1
|
|
fi
|
|
''
|
|
);
|
|
};
|
|
};
|
|
systemd.services.libvirtd = lib.mkIf cfg.lookingGlass.enable {
|
|
after = [ "kvmfr.service" ];
|
|
requires = [ "kvmfr.service" ];
|
|
};
|
|
})
|
|
(lib.mkIf cfg.hugepages.enable {
|
|
boot.kernelParams = [ "hugepagesz=2M" "hugepages=64" ];
|
|
})
|
|
]);
|
|
};
|
|
}
|