Add NixVirt support and GPU passthrough configuration
This commit is contained in:
@@ -17,6 +17,7 @@
|
||||
self.nixosModules.systemIdeapadMrubyOverlay
|
||||
self.nixosModules.systemYtDlpTelequebecPatch
|
||||
self.nixosModules.systemGaming
|
||||
self.nixosModules.systemGpuPassthru
|
||||
self.nixosModules.systemUconsoleKernelBuilder
|
||||
self.nixosModules.systemLibrepods
|
||||
self.nixosModules.systemPalera1n
|
||||
|
||||
@@ -0,0 +1,383 @@
|
||||
{ ... }: {
|
||||
flake.nixosModules.systemGpuPassthru =
|
||||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
cfg = config.chiasson.system.gpuPassthru;
|
||||
vfioIds = lib.concatStringsSep "," cfg.pciIds;
|
||||
# Must match the Windows host installer build (B7-244-4bb2c58f).
|
||||
lgRev = "4bb2c58fb6d0df9e863ad45924dd4decc7e9cf4e";
|
||||
lgVersion = "B7-244-4bb2c58f";
|
||||
# Tarball hashes omit submodules; fetchSubmodules needs the git tree hash.
|
||||
lgHash = "sha256-Dc0sFnJVM0xJ0FfIQqGQKxFUj6mhp3qgB772h3Dgowc=";
|
||||
# https://wiki.nixos.org/wiki/Looking_Glass — systemd 258+ needs namespaces=[]
|
||||
# and an explicit ACL; seccomp can block mmap on /dev/kvmfr0.
|
||||
lgQemuVerbatimConfig = ''
|
||||
namespaces = []
|
||||
seccomp_sandbox = 0
|
||||
security_driver = "none"
|
||||
security_default_confined = 0
|
||||
security_require_confined = 0
|
||||
cgroup_device_acl = [
|
||||
"/dev/null", "/dev/full", "/dev/zero",
|
||||
"/dev/random", "/dev/urandom",
|
||||
"/dev/ptmx", "/dev/kvm", "/dev/kqemu",
|
||||
"/dev/rtc", "/dev/hpet",
|
||||
"/dev/vfio/", "/dev/vfio/vfio",
|
||||
"/dev/kvmfr0", "c 234:* rwm",
|
||||
"/dev/disk/", "/dev/disk/by-id/",
|
||||
"/dev/bus/usb/",
|
||||
]
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.chiasson.system.gpuPassthru = {
|
||||
enable = lib.mkEnableOption ''
|
||||
Bind passthrough GPU(s) to VFIO, run the host on Intel iGPU, and enable libvirt +
|
||||
Looking Glass for a Windows gaming VM.
|
||||
|
||||
Prerequisites (manual, one-time):
|
||||
- BIOS: VT-d/IOMMU on, Above 4G Decoding / Resizable BAR on
|
||||
- Move host monitors to motherboard (Intel) ports; dummy HDMI/DP plug on passthrough GPU
|
||||
- Wipe the VM disk before first boot: `wipefs -a <windowsDisk>`
|
||||
- Verify IOMMU groups after switch: GPU functions should be isolated
|
||||
|
||||
Rollout: leave `enable = false` until cabling and disk backup are done, then rebuild and switch.
|
||||
'';
|
||||
|
||||
pciIds = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "10de:1f07" "10de:10f9" "10de:1ada" "10de:1adb" ];
|
||||
description = ''
|
||||
PCI vendor:device IDs to bind to vfio-pci (all GPU functions: VGA, audio, USB, etc.).
|
||||
'';
|
||||
};
|
||||
|
||||
windowsDisk = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "/dev/disk/by-id/ata-CT1000MX500SSD1_1830E14B054A";
|
||||
description = ''
|
||||
Whole-disk by-id path for the Windows VM (raw block passthrough). Must not be mounted by the host.
|
||||
'';
|
||||
};
|
||||
|
||||
lookingGlass = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Load the kvmfr kernel module and grant QEMU access to /dev/kvmfr0 for Looking Glass
|
||||
(required for NVIDIA VFIO; plain /dev/shm IVSHMEM causes DMA mapping failures).
|
||||
'';
|
||||
};
|
||||
|
||||
shmSizeMiB = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 32;
|
||||
description = ''
|
||||
kvmfr shared memory size (MiB). Must match `kvmfr.static_size_mb` and the VM
|
||||
`memory-backend-file` size in the domain XML.
|
||||
'';
|
||||
};
|
||||
|
||||
captureOnFocus = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Grab keyboard/mouse when the Looking Glass window is focused. Set `false` to
|
||||
keep input on the Linux host while viewing the guest.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
vm = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Declaratively define the Windows VM via NixVirt (host module must import NixVirt and set
|
||||
`virtualisation.libvirt.connections`).
|
||||
'';
|
||||
};
|
||||
|
||||
name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "win11";
|
||||
description = "libvirt domain name.";
|
||||
};
|
||||
|
||||
memoryGiB = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 16;
|
||||
description = "VM RAM in GiB (informational; domain XML is authoritative).";
|
||||
};
|
||||
|
||||
vcpus = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 8;
|
||||
description = "vCPU count (informational; domain XML is authoritative).";
|
||||
};
|
||||
|
||||
installMode = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Use install domain (ISO, no GPU). Set `false` after Windows is installed.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
hugepages.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Reserve 2 MiB hugepages for Looking Glass IVSHMEM (enable if frame drops).";
|
||||
};
|
||||
|
||||
libvirtUsers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Users added to libvirtd, kvm, and disk groups.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable (lib.mkMerge [
|
||||
{
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.pciIds != [ ];
|
||||
message = "chiasson.system.gpuPassthru: pciIds must list every passthrough GPU function.";
|
||||
}
|
||||
{
|
||||
assertion = cfg.windowsDisk != "";
|
||||
message = "chiasson.system.gpuPassthru: windowsDisk must be set to a /dev/disk/by-id/ path.";
|
||||
}
|
||||
{
|
||||
assertion = cfg.libvirtUsers != [ ];
|
||||
message = "chiasson.system.gpuPassthru: libvirtUsers must name at least one user.";
|
||||
}
|
||||
];
|
||||
|
||||
boot.kernelParams = [
|
||||
"intel_iommu=on"
|
||||
"iommu=pt"
|
||||
"kvm.ignore_msrs=1"
|
||||
"vfio-pci.ids=${vfioIds}"
|
||||
];
|
||||
|
||||
boot.initrd.kernelModules = [
|
||||
"vfio_pci"
|
||||
"vfio"
|
||||
"vfio_iommu_type1"
|
||||
];
|
||||
|
||||
boot.blacklistedKernelModules = [
|
||||
"nouveau"
|
||||
"nvidia"
|
||||
"nvidia_drm"
|
||||
"nvidia_modeset"
|
||||
"i2c_nvidia_gpu"
|
||||
"mt7921e"
|
||||
"mt7925e"
|
||||
];
|
||||
|
||||
boot.extraModprobeConfig = ''
|
||||
softdep nvidia pre: vfio-pci
|
||||
softdep nvidia_drm pre: vfio-pci
|
||||
softdep nouveau pre: vfio-pci
|
||||
options vfio_iommu_type1 allow_unsafe_interrupts=1
|
||||
'';
|
||||
|
||||
services.xserver.videoDrivers = lib.mkForce [ "modesetting" ];
|
||||
|
||||
hardware.nvidia-container-toolkit.enable = lib.mkForce false;
|
||||
chiasson.system.gaming.sunshine.cudaSupport = lib.mkForce false;
|
||||
|
||||
# libvirt enables ZFS storage pools on Linux by default, which pulls in
|
||||
# zfs-user (currently broken to build on nixos-unstable). Raw block disk
|
||||
# passthrough does not need libvirt ZFS support.
|
||||
nixpkgs.overlays =
|
||||
[
|
||||
(final: prev: {
|
||||
libvirt = prev.libvirt.override { enableZfs = false; };
|
||||
})
|
||||
]
|
||||
++ lib.optionals cfg.lookingGlass.enable [
|
||||
(final: prev: {
|
||||
looking-glass-client = prev.looking-glass-client.overrideAttrs (old: {
|
||||
version = lgVersion;
|
||||
src = prev.fetchFromGitHub {
|
||||
owner = "gnif";
|
||||
repo = "LookingGlass";
|
||||
rev = lgRev;
|
||||
hash = lgHash;
|
||||
fetchSubmodules = true;
|
||||
};
|
||||
patches = old.patches;
|
||||
postUnpack = ''
|
||||
echo ${lgRev} > $sourceRoot/VERSION
|
||||
export sourceRoot="$sourceRoot/client"
|
||||
'';
|
||||
});
|
||||
})
|
||||
];
|
||||
|
||||
programs.virt-manager.enable = true;
|
||||
virtualisation.spiceUSBRedirection.enable = true;
|
||||
|
||||
virtualisation.libvirtd = {
|
||||
enable = true;
|
||||
onBoot = "ignore";
|
||||
onShutdown = "shutdown";
|
||||
qemu = {
|
||||
package = pkgs.qemu_kvm;
|
||||
# GPU + raw block passthrough needs root (or fragile per-device ACLs).
|
||||
runAsRoot = true;
|
||||
swtpm.enable = true;
|
||||
verbatimConfig = lib.mkIf cfg.lookingGlass.enable (lib.mkForce lgQemuVerbatimConfig);
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.libvirtd.restartTriggers = [
|
||||
"${pkgs.qemu_kvm}/bin/qemu-system-x86_64"
|
||||
];
|
||||
|
||||
environment.systemPackages =
|
||||
with pkgs;
|
||||
[
|
||||
virt-manager
|
||||
qemu_kvm
|
||||
OVMF
|
||||
edk2
|
||||
looking-glass-client
|
||||
swtpm
|
||||
]
|
||||
++ lib.optionals cfg.lookingGlass.enable [
|
||||
(writeShellScriptBin "looking-glass-toggle" ''
|
||||
set -euo pipefail
|
||||
pkill -f 'looking-glass-client' || true
|
||||
exec looking-glass-client -F -m 97
|
||||
'')
|
||||
];
|
||||
|
||||
# B7 defaults to /dev/kvmfr0; pin SPICE to the fixed port in the domain XMLs.
|
||||
environment.etc."looking-glass-client.ini" = lib.mkIf cfg.lookingGlass.enable (lib.mkForce {
|
||||
text = ''
|
||||
[app]
|
||||
shmFile=/dev/kvmfr0
|
||||
allowDMA=yes
|
||||
|
||||
[win]
|
||||
fullScreen=yes
|
||||
|
||||
[spice]
|
||||
enable=yes
|
||||
host=127.0.0.1
|
||||
port=5900
|
||||
input=yes
|
||||
audio=yes
|
||||
clipboard=yes
|
||||
|
||||
[input]
|
||||
captureOnFocus=${lib.boolToString cfg.lookingGlass.captureOnFocus}
|
||||
escapeKey=97
|
||||
'';
|
||||
});
|
||||
|
||||
users.groups.libvirtd.members = cfg.libvirtUsers;
|
||||
users.users =
|
||||
lib.genAttrs cfg.libvirtUsers (_: {
|
||||
extraGroups = [
|
||||
"libvirtd"
|
||||
"kvm"
|
||||
"disk"
|
||||
];
|
||||
})
|
||||
// {
|
||||
qemu-libvirtd.extraGroups = [
|
||||
"kvm"
|
||||
"disk"
|
||||
];
|
||||
};
|
||||
}
|
||||
{
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/lib/libvirt/boot 0755 root root -"
|
||||
];
|
||||
}
|
||||
(lib.mkIf cfg.lookingGlass.enable {
|
||||
boot.extraModulePackages = lib.mkAfter [ config.boot.kernelPackages.kvmfr ];
|
||||
boot.initrd.kernelModules = lib.mkAfter [ "kvmfr" ];
|
||||
boot.kernelModules = lib.mkAfter [ "kvmfr" ];
|
||||
boot.kernelParams = lib.mkAfter [
|
||||
"kvmfr.static_size_mb=${toString cfg.lookingGlass.shmSizeMiB}"
|
||||
];
|
||||
|
||||
boot.extraModprobeConfig = lib.mkAfter ''
|
||||
options kvmfr static_size_mb=${toString cfg.lookingGlass.shmSizeMiB}
|
||||
'';
|
||||
|
||||
services.udev.packages = [
|
||||
(pkgs.writeTextFile {
|
||||
name = "kvmfr-udev";
|
||||
text = ''
|
||||
SUBSYSTEM=="kvmfr", GROUP="kvm", MODE="0660", TAG+="uaccess"
|
||||
'';
|
||||
destination = "/etc/udev/rules.d/70-kvmfr.rules";
|
||||
})
|
||||
];
|
||||
|
||||
# insmod from the nix store: extraModulePackages only lands in
|
||||
# /run/booted-system/kernel-modules after a reboot.
|
||||
systemd.services.kvmfr = {
|
||||
description = "Looking Glass kvmfr module";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [
|
||||
"libvirtd.service"
|
||||
"libvirtd-config.service"
|
||||
"libvirt-guests.service"
|
||||
];
|
||||
after = [ "systemd-modules-load.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = pkgs.writeShellScript "kvmfr-start" (
|
||||
let
|
||||
kvmfrKo = "${config.boot.kernelPackages.kvmfr}/lib/modules/${config.boot.kernelPackages.kernel.modDirVersion}/kernel/drivers/misc/kvmfr.ko";
|
||||
shmMiB = toString cfg.lookingGlass.shmSizeMiB;
|
||||
in
|
||||
''
|
||||
set -euo pipefail
|
||||
if [ -e /dev/kvmfr0 ] && [ ! -c /dev/kvmfr0 ]; then
|
||||
rm -f /dev/kvmfr0
|
||||
fi
|
||||
if [ -c /dev/kvmfr0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
if ${pkgs.kmod}/bin/lsmod | ${pkgs.gnugrep}/bin/grep -q '^kvmfr '; then
|
||||
${pkgs.kmod}/bin/rmmod kvmfr
|
||||
fi
|
||||
if [ ! -f ${kvmfrKo} ]; then
|
||||
echo "kvmfr: module missing at ${kvmfrKo}" >&2
|
||||
exit 1
|
||||
fi
|
||||
${pkgs.kmod}/bin/insmod ${kvmfrKo} static_size_mb=${shmMiB}
|
||||
if [ ! -c /dev/kvmfr0 ]; then
|
||||
echo "kvmfr: /dev/kvmfr0 not created (static_size_mb=${shmMiB})" >&2
|
||||
exit 1
|
||||
fi
|
||||
''
|
||||
);
|
||||
};
|
||||
};
|
||||
systemd.services.libvirtd = lib.mkIf cfg.lookingGlass.enable {
|
||||
after = [ "kvmfr.service" ];
|
||||
requires = [ "kvmfr.service" ];
|
||||
};
|
||||
})
|
||||
(lib.mkIf cfg.hugepages.enable {
|
||||
boot.kernelParams = [ "hugepagesz=2M" "hugepages=64" ];
|
||||
})
|
||||
]);
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user