Switch ephemeral overlay from qcow2 to raw sparse image
qcow2 causes O_DIRECT failures on ext4 due to crosvm doing unaligned access when parsing the qcow2 header. Since we don't use any qcow2 features (the disk is created fresh and deleted on stop), a raw sparse file via truncate works just as well and also removes the qemu package dependency from the VM service. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
b8646f3020
commit
0a07f7f14e
6 changed files with 23 additions and 23 deletions
|
|
@ -58,7 +58,7 @@ The configured user can manage VM services via polkit (no sudo required for `vm-
|
|||
- `networking.nix` — TAP interfaces, host nftables for `netvm = "host"`, guest network config
|
||||
- `netvm.nix` — auto VM-to-VM and VM-to-host network links
|
||||
- `pci.nix` — VFIO PCI passthrough setup
|
||||
- `overlay.nix` — root overlay (qcow2/tmpfs) setup
|
||||
- `overlay.nix` — root overlay (raw/tmpfs) setup
|
||||
- `assertions.nix` — validation checks
|
||||
- `package.nix` — final package derivation
|
||||
- `lib/helpers.nix` — crosvm argument building helpers
|
||||
|
|
@ -105,7 +105,7 @@ See README.md for full usage details and options.
|
|||
- **Socket activation chain**: command socket → proxy template service → VM service → crosvm
|
||||
- **Offline-by-default networking**: VMs have no network unless `network.interfaces` or `network.netvm` is configured.
|
||||
- **Host netvm**: `network.netvm = "host"` routes VM traffic through the host directly (TAP with hostAddress, host nftables NAT). No bridge or netvm VM needed.
|
||||
- **Root overlay**: qcow2-backed (default) or tmpfs-backed overlayfs upper layer. Ephemeral — created at VM start, deleted at stop.
|
||||
- **Root overlay**: raw disk-backed (default) or tmpfs-backed overlayfs upper layer. Ephemeral — created at VM start, deleted at stop.
|
||||
- **Session bind**: GPU-enabled VMs (default) are tied to the desktop session via per-VM systemd user services bound to `graphical-session.target`. For `autoStart` GPU VMs, the session-bind service also starts the VM on login. Non-GPU `autoStart` VMs start at `multi-user.target` (boot).
|
||||
|
||||
### Gotchas
|
||||
|
|
|
|||
|
|
@ -191,8 +191,8 @@ There are a lot of configuration options but you don't really need to touch most
|
|||
| `vhostUser` | list of attrsets | `[]` | Manual vhost-user devices |
|
||||
| `crosvm.logLevel` | string or null | null | Per-VM log level override (uses global if null) |
|
||||
| `crosvm.extraArgs` | list of strings | `[]` | Per-VM extra args (appended to global crosvm.extraArgs) |
|
||||
| `rootOverlay.type` | `"qcow2"` or `"tmpfs"` | `"qcow2"` | Overlay upper layer: disk-backed (qcow2) or RAM-backed (tmpfs) |
|
||||
| `rootOverlay.size` | string | `"10G"` | Max ephemeral disk size (qcow2 only). Parsed by qemu-img |
|
||||
| `rootOverlay.type` | `"raw"` or `"tmpfs"` | `"raw"` | Overlay upper layer: disk-backed (raw) or RAM-backed (tmpfs) |
|
||||
| `rootOverlay.size` | string | `"10G"` | Max ephemeral disk size (raw only). Parsed by truncate |
|
||||
| `crosvm.extraRunArgs` | list of strings | `[]` | Per-VM extra run args (appended to global crosvm.extraRunArgs) |
|
||||
|
||||
### netvm (auto VM-to-VM links)
|
||||
|
|
@ -513,7 +513,7 @@ Interface names are user-specified via `network.interfaces` attrset keys. Names
|
|||
|
||||
Each NixOS VM gets:
|
||||
- An erofs rootfs image with packages baked in (compressed, read-only)
|
||||
- Overlayfs root (read-only erofs lower + ephemeral qcow2 upper by default, tmpfs fallback)
|
||||
- Overlayfs root (read-only erofs lower + ephemeral raw disk upper by default, tmpfs fallback)
|
||||
- Wayland proxy for GPU passthrough (wayland-proxy-virtwl or sommelier)
|
||||
- Session setup via `vmsilo-session-setup` (imports display variables into user manager, starts `graphical-session.target`)
|
||||
- Socket-activated command listener (`vsock-cmd.socket` + `vsock-cmd@.service`, user services gated on `graphical-session.target`)
|
||||
|
|
|
|||
|
|
@ -273,17 +273,17 @@ let
|
|||
options = {
|
||||
type = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"qcow2"
|
||||
"raw"
|
||||
"tmpfs"
|
||||
];
|
||||
default = "qcow2";
|
||||
description = "Overlay upper layer type: 'qcow2' (disk-backed, default) or 'tmpfs' (RAM-backed).";
|
||||
default = "raw";
|
||||
description = "Overlay upper layer type: 'raw' (disk-backed, default) or 'tmpfs' (RAM-backed).";
|
||||
};
|
||||
|
||||
size = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10G";
|
||||
description = "Maximum size of ephemeral disk (qcow2 only). Parsed by qemu-img create.";
|
||||
description = "Maximum size of ephemeral disk (raw only). Parsed by truncate.";
|
||||
example = "20G";
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -140,22 +140,22 @@ let
|
|||
|
||||
additionalDisksArgs = lib.concatMapStringsSep " " formatBlockArg vm.additionalDisks;
|
||||
|
||||
# Ephemeral overlay disk (qcow2 mode only)
|
||||
ephemeralDiskPath = "/var/lib/vmsilo/${vm.name}-ephemeral.qcow2";
|
||||
# Ephemeral overlay disk (raw mode only)
|
||||
ephemeralDiskPath = "/var/lib/vmsilo/${vm.name}-ephemeral.raw";
|
||||
ephemeralDiskId = "ephemeral";
|
||||
ephemeralDiskConfig = {
|
||||
path = ephemeralDiskPath;
|
||||
ro = false;
|
||||
id = ephemeralDiskId;
|
||||
};
|
||||
ephemeralDiskArg = lib.optionalString (vm.rootOverlay.type == "qcow2") (
|
||||
ephemeralDiskArg = lib.optionalString (vm.rootOverlay.type == "raw") (
|
||||
formatBlockArg ephemeralDiskConfig
|
||||
);
|
||||
|
||||
# Kernel param for overlay type
|
||||
rootOverlayKernelParam =
|
||||
if vm.rootOverlay.type == "qcow2" then
|
||||
''-p "vmsilo.rootOverlay=qcow2,${ephemeralDiskId}"''
|
||||
if vm.rootOverlay.type == "raw" then
|
||||
''-p "vmsilo.rootOverlay=raw,${ephemeralDiskId}"''
|
||||
else
|
||||
''-p "vmsilo.rootOverlay=tmpfs"'';
|
||||
|
||||
|
|
|
|||
|
|
@ -120,21 +120,21 @@ in
|
|||
};
|
||||
serviceConfig =
|
||||
let
|
||||
ephemeralPath = "/var/lib/vmsilo/${vm.name}-ephemeral.qcow2";
|
||||
createQcow2 = pkgs.writeShellScript "create-ephemeral-${vm.name}" ''
|
||||
${pkgs.qemu}/bin/qemu-img create -f qcow2 ${ephemeralPath} ${vm.rootOverlay.size}
|
||||
ephemeralPath = "/var/lib/vmsilo/${vm.name}-ephemeral.raw";
|
||||
createEphemeral = pkgs.writeShellScript "create-ephemeral-${vm.name}" ''
|
||||
truncate -s ${vm.rootOverlay.size} ${ephemeralPath}
|
||||
'';
|
||||
deleteQcow2 = pkgs.writeShellScript "delete-ephemeral-${vm.name}" ''
|
||||
deleteEphemeral = pkgs.writeShellScript "delete-ephemeral-${vm.name}" ''
|
||||
rm -f ${ephemeralPath}
|
||||
'';
|
||||
startPreScripts = lib.optionals (vm.rootOverlay.type == "qcow2") [ "${createQcow2}" ];
|
||||
startPreScripts = lib.optionals (vm.rootOverlay.type == "raw") [ "${createEphemeral}" ];
|
||||
cleanupSocket = pkgs.writeShellScript "cleanup-socket-${vm.name}" ''
|
||||
rm -f /run/vmsilo/${vm.name}-crosvm-control.socket
|
||||
'';
|
||||
stopPostScripts = [
|
||||
"${cleanupSocket}"
|
||||
]
|
||||
++ lib.optionals (vm.rootOverlay.type == "qcow2") [ "${deleteQcow2}" ];
|
||||
++ lib.optionals (vm.rootOverlay.type == "raw") [ "${deleteEphemeral}" ];
|
||||
in
|
||||
{
|
||||
Type = "simple";
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Boot configuration and overlayfs root setup for guest VMs
|
||||
#
|
||||
# - Systemd stage 1 initrd with virtio kernel modules
|
||||
# - Overlayfs root: read-only erofs lower + tmpfs/qcow2 upper
|
||||
# - Overlayfs root: read-only erofs lower + tmpfs/raw upper
|
||||
{ pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
|
|
@ -105,8 +105,8 @@ in
|
|||
tmpfs)
|
||||
mount -t tmpfs -o mode=0755 tmpfs /sysroot/.overlay-rw
|
||||
;;
|
||||
qcow2,*)
|
||||
DEVICE_ID="''${OVERLAY_TYPE#qcow2,}"
|
||||
raw,*)
|
||||
DEVICE_ID="''${OVERLAY_TYPE#raw,}"
|
||||
DEVICE="/dev/disk/by-id/virtio-''${DEVICE_ID}"
|
||||
|
||||
# Wait for device with timeout
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue