WIP: Add rootfs-nixos package for NixOS-based VM images

Adds a new rootfs-nixos/ package that builds a NixOS system into a
qcow2 image with overlayfs root (read-only ext4 + tmpfs upper).
This commit is contained in:
Davíð Steinn Geirsson 2026-02-03 20:51:37 +00:00
parent b46e11d551
commit c50b309807
4 changed files with 536 additions and 4 deletions

View file

@ -37,6 +37,16 @@ let
];
};
# NixOS-based root filesystem (self-contained qcow2 image)
rootfsNixos = callPackage (import ./rootfs-nixos) {
inherit wayland-proxy-virtwl;
guestPrograms = [
kdePackages.konsole
firefox
xwayland
];
};
# Modules to put in the initrd for guests.
# At the least, we need to load virtiofs here in order to mount /nix/store from the host.
# This avoids the need to compile a custom kernel, which takes ages.
@ -125,7 +135,7 @@ let
mktuntap = callPackage (import ./mktuntap) {};
# Create a vs* script to launch a VM.
# Create a vs* script to launch a VM (s6-based rootfs).
mkStart = { name, command ? "", cpus ? 2, network ? true, hostNumber, rootDev, disks, memory ? 4096 }:
{ inherit name;
value =
@ -179,6 +189,58 @@ let
'';
};
# Create a vs* script to launch a NixOS-based VM (uses self-contained qcow2 image).
mkStartNixos = { name, command ? "", cpus ? 2, network ? true, hostNumber, disks, memory ? 4096 }:
{ inherit name;
value =
''
#!${execline}/bin/execlineb -s0
foreground { umount /run/user/1000/doc }
backtick command { pipeline { echo -n $@ } base64 -w0 }
importas -iu command command
importas -iu XDG_RUNTIME_DIR XDG_RUNTIME_DIR
importas -iu WAYLAND_DISPLAY WAYLAND_DISPLAY
'' + lib.optionalString network ''
${mktuntap}/bin/mktuntap -i tap${name} -pvB 3
'' + ''
${pkgs.s6}/bin/s6-softlimit -l 0
"${lib.getBin crosvm}/bin/crosvm" run
-m ${toString memory}
--initrd=${rootfsNixos}/initrd
--serial=hardware=virtio-console
${builtins.concatStringsSep "\n" disks}
--shared-dir "/home/david/vms/shared/${name}:mtdshared:type=fs"
-p "init=${rootfsNixos.config.system.build.toplevel}/init"
-p "net.ifnames=0"
-p "spectrumcmd=$command"
-p "spectrumname=${name}"
'' +
lib.optionalString network (
let
ip = "10.0.0.${toString hostNumber}";
gw = "10.0.0.${toString (hostNumber - 1)}";
ipv6 = "fd4d:06ff:48e4:${toString (hostNumber - 1)}::2/48";
gwv6 = "fd4d:06ff:48e4:${toString (hostNumber - 1)}::1";
in
''
-p "spectrumip=${ip}"
-p "spectrumgw=${gw}"
-p "spectrumip6=${ipv6}"
-p "spectrumgw6=${gwv6}"
--tap-fd 3
''
) + ''
--cid ${toString hostNumber}
-p root=/dev/vda
--seccomp-log-failures
--cpus ${toString cpus}
--gpu=context-types=cross-domain:virgl2
-s $\{XDG_RUNTIME_DIR}"/crosvm-${name}.sock"
--wayland-sock "$\{XDG_RUNTIME_DIR}/$\{WAYLAND_DISPLAY}"
"${rootfsNixos}/bzImage"
'';
};
# Configuration for each VM.
vms = [
{
@ -246,10 +308,25 @@ let
}
];
# NixOS-based VMs (use self-contained qcow2 images with systemd)
nixosVms = [
{
name = "fulltest";
memory = 4096;
hostNumber = 15;
cpus = 4;
disks = [
"--disk ${rootfsNixos}/nixos.qcow2"
];
}
];
# The list of vs* scripts.
vmScripts = map mkStart vms;
nixosVmScripts = map mkStartNixos nixosVms;
allVmScripts = vmScripts ++ nixosVmScripts;
attrs = builtins.listToAttrs (map ({name, value}: { name = "${name}Script"; inherit value; }) vmScripts);
attrs = builtins.listToAttrs (map ({name, value}: { name = "${name}Script"; inherit value; }) allVmScripts);
# Systemd unit file to also run the proxy on the host. This fixes some DPI problems with Xwayland.
xwaylandService = ''
@ -293,12 +370,13 @@ in
runCommand "qubes-lite" (attrs // { inherit xwaylandService; })
''
mkdir -p "$out/bin"
${builtins.concatStringsSep "\n" (map ({name, value}: ''echo -n "$'' + ''${name}Script" > "$out/bin/vs${name}"'') vmScripts)}
${builtins.concatStringsSep "\n" (map spawnApp vms)}
${builtins.concatStringsSep "\n" (map ({name, value}: ''echo -n "$'' + ''${name}Script" > "$out/bin/vs${name}"'') allVmScripts)}
${builtins.concatStringsSep "\n" (map spawnApp (vms ++ nixosVms))}
chmod +x "$out/bin/"*
ln -s "${wayland-proxy-virtwl}/bin/wayland-proxy-virtwl" "$out/bin/wayland-proxy-virtwl"
mkdir -p "$out/share/systemd/user"
echo -n "$xwaylandService" > "$out/share/systemd/user/xwayland-proxy.service"
ln -s ${rootfs} "$out/rootfs-gc-keep"
ln -s ${rootfsNixos} "$out/rootfs-nixos-gc-keep"
ln -s ${crosvm}/bin/crosvm "$out/bin/crosvm"
''

View file

@ -14,6 +14,8 @@
outputs = { self, nixpkgs, wayland-proxy-virtwl, crosvm }:
let
eachSystem = nixpkgs.lib.genAttrs ["x86_64-linux" "aarch64-linux"];
# Build the main qubes-lite package (s6-based rootfs)
make = system: guestPrograms:
let pkgs = nixpkgs.legacyPackages.${system}; in
pkgs.callPackage (import ./default.nix) {
@ -21,12 +23,24 @@
wayland-proxy-virtwl = wayland-proxy-virtwl.packages.${system}.default;
crosvm = crosvm.packages.${system}.default;
};
# Build NixOS-based rootfs as qcow2 image
makeRootfsNixos = system: { guestPrograms ? [], guestConfig ? {} }:
let pkgs = nixpkgs.legacyPackages.${system}; in
pkgs.callPackage (import ./rootfs-nixos) {
inherit guestPrograms guestConfig;
wayland-proxy-virtwl = wayland-proxy-virtwl.packages.${system}.default;
};
in
{
packages = eachSystem(system: {
default = make system [];
rootfs-nixos = makeRootfsNixos system {};
});
# Helper function for building custom NixOS rootfs
lib.makeRootfsNixos = makeRootfsNixos;
nixosModules.default = { config, pkgs, lib, ... }: {
options = {
programs.qubes-lite.guestPrograms = lib.mkOption {

View file

@ -0,0 +1,366 @@
# NixOS configuration for qubes-lite guest VMs
#
# Features:
# - Systemd-based stage 1 initrd
# - Overlayfs root (read-only ext4 lower + tmpfs upper)
# - wayland-proxy-virtwl for GPU passthrough
# - Network configuration from kernel command line
# - vsock command listener for host communication
#
{ config, pkgs, lib, wayland-proxy-virtwl, ... }:
{
#############################################################################
# Boot Configuration - Systemd Stage 1
#############################################################################
boot.initrd.systemd.enable = true;
# Kernel modules needed in initrd for virtio devices
boot.initrd.availableKernelModules = [
"virtio_pci"
"virtio_blk"
"virtio_net"
"virtio_console"
"virtio_gpu"
"virtiofs"
"virtio_balloon"
"virtio_rng"
"ext4"
"overlay"
"vsock"
"vmw_vsock_virtio_transport"
"fuse"
];
# Load these modules at boot
boot.initrd.kernelModules = [ "overlay" ];
# No bootloader - crosvm uses direct kernel boot
boot.loader.grub.enable = false;
# Disable remount service - overlay filesystems cannot be remounted
systemd.services.systemd-remount-fs.serviceConfig.ExecStart = [
"" # Clear the default ExecStart
"${pkgs.coreutils}/bin/true" # Do nothing successfully
];
#############################################################################
# Overlayfs Root Configuration
#############################################################################
# Dummy root filesystem entry - actual mount is handled by initrd
# We use "none" device to prevent NixOS from generating conflicting mounts
fileSystems."/" = {
device = "overlay";
fsType = "overlay";
options = [ "defaults" ];
# Don't generate initrd mount - we handle it manually
neededForBoot = false;
};
# Custom sysroot.mount for overlay in initrd
boot.initrd.systemd.mounts = [{
where = "/sysroot";
what = "overlay";
type = "overlay";
options = "lowerdir=/sysroot/.overlay-lower,upperdir=/sysroot/.overlay-rw/upper,workdir=/sysroot/.overlay-rw/work";
wantedBy = [ "initrd-root-fs.target" ];
before = [ "initrd-root-fs.target" ];
after = [ "prepare-overlay.service" ];
requires = [ "prepare-overlay.service" ];
unitConfig = {
DefaultDependencies = false;
RequiresMountsFor = "/sysroot/.overlay-lower /sysroot/.overlay-rw";
};
}];
# Prepare the overlay mounts before sysroot.mount
boot.initrd.systemd.services.prepare-overlay = {
description = "Prepare overlay filesystems";
wantedBy = [ "initrd-root-fs.target" ];
before = [ "sysroot.mount" ];
after = [ "dev-vda.device" ];
requires = [ "dev-vda.device" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
# Create mount points
mkdir -p /sysroot/.overlay-lower /sysroot/.overlay-rw
# Mount ext4 as read-only lower layer
mount -t ext4 -o ro /dev/vda /sysroot/.overlay-lower
# Mount tmpfs for writable upper layer
mount -t tmpfs -o mode=0755 tmpfs /sysroot/.overlay-rw
# Create upper and work directories
mkdir -p /sysroot/.overlay-rw/upper /sysroot/.overlay-rw/work
'';
};
#############################################################################
# User Configuration
#############################################################################
boot.initrd.systemd.emergencyAccess = true;
# Temp for testing
users.users.root.password = "foo";
users.users.user = {
isNormalUser = true;
uid = 1000;
group = "user";
extraGroups = [ "wheel" "video" "audio" "input" ];
# Home directory - user can configure persistent storage via guestConfig
home = "/home/user";
createHome = true;
};
users.groups.user.gid = 1000;
# Passwordless sudo for user
security.sudo.wheelNeedsPassword = false;
#############################################################################
# Network Configuration (from kernel parameters)
#############################################################################
networking.useDHCP = false;
networking.useNetworkd = true;
networking.usePredictableInterfaceNames = false; # Use eth0 instead of enp0s7
# Set hostname from kernel param spectrumname
systemd.services.set-hostname = {
description = "Set hostname from kernel command line";
wantedBy = [ "multi-user.target" ];
before = [ "network.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = pkgs.writeShellScript "set-hostname" ''
for param in $(cat /proc/cmdline); do
case "$param" in
spectrumname=*)
hostname="''${param#spectrumname=}"
${pkgs.hostname}/bin/hostname "$hostname"
;;
esac
done
'';
};
};
# Configure network from kernel parameters
systemd.services.configure-network = {
description = "Configure network from kernel command line";
wantedBy = [ "multi-user.target" ];
after = [ "systemd-networkd.service" "sys-subsystem-net-devices-eth0.device" ];
wants = [ "sys-subsystem-net-devices-eth0.device" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = pkgs.writeShellScript "configure-network" ''
SPECTRUMIP=""
SPECTRUMGW=""
SPECTRUMIP6=""
SPECTRUMGW6=""
for param in $(cat /proc/cmdline); do
case "$param" in
spectrumip=*) SPECTRUMIP="''${param#spectrumip=}" ;;
spectrumgw=*) SPECTRUMGW="''${param#spectrumgw=}" ;;
spectrumip6=*) SPECTRUMIP6="''${param#spectrumip6=}" ;;
spectrumgw6=*) SPECTRUMGW6="''${param#spectrumgw6=}" ;;
esac
done
# Configure loopback
${pkgs.iproute2}/bin/ip link set lo up
${pkgs.iproute2}/bin/ip addr add 127.0.0.1/8 dev lo 2>/dev/null || true
# Configure eth0 if IP provided
if [ -n "$SPECTRUMIP" ] && [ "$SPECTRUMIP" != "-" ]; then
${pkgs.iproute2}/bin/ip addr add "$SPECTRUMIP/31" dev eth0
${pkgs.iproute2}/bin/ip link set eth0 up
${pkgs.iproute2}/bin/ip route add default via "$SPECTRUMGW"
if [ -n "$SPECTRUMIP6" ]; then
${pkgs.iproute2}/bin/ip -6 addr add "$SPECTRUMIP6" dev eth0
${pkgs.iproute2}/bin/ip -6 route add default via "$SPECTRUMGW6"
fi
fi
'';
};
};
#############################################################################
# wayland-proxy-virtwl Systemd Service
#############################################################################
systemd.services.wayland-proxy-virtwl = {
description = "Wayland proxy for virtio-gpu";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
path = [ pkgs.xwayland ];
environment = {
XDG_RUNTIME_DIR = "/run/user/1000";
HOME = "/home/user";
};
serviceConfig = {
Type = "simple";
User = "user";
Group = "user";
ExecStartPre = "+${pkgs.writeShellScript "setup-runtime-dir" ''
mkdir -p /run/user/1000
chown user:user /run/user/1000
chmod 700 /run/user/1000
''}";
ExecStart = pkgs.writeShellScript "start-wayland-proxy" ''
# Extract VM name from kernel command line
VMNAME="vm"
for param in $(cat /proc/cmdline); do
case "$param" in
spectrumname=*) VMNAME="''${param#spectrumname=}" ;;
esac
done
exec ${wayland-proxy-virtwl}/bin/wayland-proxy-virtwl \
--virtio-gpu \
--tag="[$VMNAME] " \
--x-display=0 \
--xrdb Xft.dpi:150 \
-v \
--log-suppress motion \
--log-ring-path /home/user/wayland.log \
--x-unscale=2 \
-- ${pkgs.dbus}/bin/dbus-launch --exit-with-session sleep infinity
'';
Restart = "on-failure";
RestartSec = "5s";
};
};
#############################################################################
# Command Execution via vsock
#############################################################################
# Listen for commands from host via vsock
systemd.services.vsock-command-listener = {
description = "Listen for commands from host via vsock";
wantedBy = [ "multi-user.target" ];
after = [ "wayland-proxy-virtwl.service" ];
environment = {
WAYLAND_DISPLAY = "wayland-0";
DISPLAY = ":0";
XDG_RUNTIME_DIR = "/run/user/1000";
HOME = "/home/user";
PATH = lib.mkForce "/run/current-system/sw/bin";
};
serviceConfig = {
Type = "simple";
User = "user";
Group = "user";
ExecStart = "${pkgs.socat}/bin/socat vsock-listen:5000,reuseaddr,fork exec:${pkgs.bashInteractive}/bin/bash";
Restart = "always";
RestartSec = "1s";
};
};
# Execute command from kernel param on boot
systemd.services.run-spectrum-cmd = {
description = "Execute command from kernel command line";
wantedBy = [ "multi-user.target" ];
after = [ "wayland-proxy-virtwl.service" ];
environment = {
WAYLAND_DISPLAY = "wayland-0";
DISPLAY = ":0";
XDG_RUNTIME_DIR = "/run/user/1000";
HOME = "/home/user";
};
path = [ pkgs.bashInteractive pkgs.coreutils ];
serviceConfig = {
Type = "oneshot";
User = "user";
Group = "user";
ExecStart = pkgs.writeShellScript "run-spectrum-cmd" ''
SPECTRUMCMD=""
for param in $(cat /proc/cmdline); do
case "$param" in
spectrumcmd=*) SPECTRUMCMD="''${param#spectrumcmd=}" ;;
esac
done
if [ -n "$SPECTRUMCMD" ]; then
echo "$SPECTRUMCMD" | base64 -d | ${pkgs.bashInteractive}/bin/bash
fi
'';
};
};
#############################################################################
# Graphics & Hardware Configuration
#############################################################################
# OpenGL/Mesa for virtio-gpu
hardware.graphics.enable = true;
# Device permissions for GPU and vsock
services.udev.extraRules = ''
KERNEL=="card0", SUBSYSTEM=="drm", MODE="0666"
KERNEL=="vsock", MODE="0666"
'';
#############################################################################
# System Configuration
#############################################################################
# Locale and timezone
i18n.defaultLocale = "en_GB.UTF-8";
time.timeZone = "Europe/London";
# Essential packages
environment.systemPackages = with pkgs; [
bashInteractive
vim
socat
pciutils
util-linux
iproute2
xwayland
xfce.xfce4-terminal
perf
util-linux
wl-clipboard
];
# D-Bus for desktop applications
services.dbus.enable = true;
# Fonts
fonts.packages = with pkgs; [
source-code-pro
dejavu_fonts
];
# Disable services we don't need
services.timesyncd.enable = false;
# System version
system.stateVersion = "25.11";
}

74
rootfs-nixos/default.nix Normal file
View file

@ -0,0 +1,74 @@
# Build a NixOS system into a qcow2 image with overlayfs root
#
# Outputs:
# qcow2 - The disk image (raw ext4, no partitions)
# kernel - The kernel for direct boot
# initrd - The initramfs for direct boot
#
{ pkgs
, lib ? pkgs.lib
, wayland-proxy-virtwl
, guestPrograms ? []
, guestConfig ? {}
}:
let
# Evaluate the NixOS configuration
nixos = pkgs.nixos ({ config, pkgs, lib, ... }: {
imports = [
./configuration.nix
guestConfig
];
# Pass wayland-proxy-virtwl to the configuration
_module.args.wayland-proxy-virtwl = wayland-proxy-virtwl;
# Guest programs included directly in image
environment.systemPackages = guestPrograms;
});
# Empty directory for overlay mount points
emptyDir = pkgs.runCommand "empty-dir" {} "mkdir $out";
# Build the disk image
diskImage = import "${pkgs.path}/nixos/lib/make-disk-image.nix" {
inherit pkgs lib;
config = nixos.config;
# Raw ext4 filesystem, no partition table
partitionTableType = "none";
format = "qcow2";
fsType = "ext4";
label = "nixos";
# No bootloader - crosvm uses direct kernel boot
installBootLoader = false;
# Size configuration
diskSize = "auto";
additionalSpace = "512M";
# Don't include channel
copyChannel = false;
# Create overlay mount point directories in the image
contents = [
{ source = emptyDir; target = "/.overlay-lower"; }
{ source = emptyDir; target = "/.overlay-rw"; }
];
};
kernel = nixos.config.system.build.kernel;
initrd = nixos.config.system.build.initialRamdisk;
in pkgs.runCommand "rootfs-nixos" {
passthru = {
inherit nixos diskImage kernel initrd;
config = nixos.config;
};
} ''
mkdir -p $out
ln -s ${diskImage}/nixos.qcow2 $out/nixos.qcow2
ln -s ${kernel}/bzImage $out/bzImage
ln -s ${initrd}/initrd $out/initrd
''