From c50b309807cfbc9f8a39646393ddc80675e15dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=AD=C3=B0=20Steinn=20Geirsson?= Date: Tue, 3 Feb 2026 20:51:37 +0000 Subject: [PATCH] WIP: Add rootfs-nixos package for NixOS-based VM images Adds a new rootfs-nixos/ package that builds a NixOS system into a qcow2 image with overlayfs root (read-only ext4 + tmpfs upper). --- default.nix | 86 +++++++- flake.nix | 14 ++ rootfs-nixos/configuration.nix | 366 +++++++++++++++++++++++++++++++++ rootfs-nixos/default.nix | 74 +++++++ 4 files changed, 536 insertions(+), 4 deletions(-) create mode 100644 rootfs-nixos/configuration.nix create mode 100644 rootfs-nixos/default.nix diff --git a/default.nix b/default.nix index 6a537b9..3895c10 100644 --- a/default.nix +++ b/default.nix @@ -37,6 +37,16 @@ let ]; }; + # NixOS-based root filesystem (self-contained qcow2 image) + rootfsNixos = callPackage (import ./rootfs-nixos) { + inherit wayland-proxy-virtwl; + guestPrograms = [ + kdePackages.konsole + firefox + xwayland + ]; + }; + # Modules to put in the initrd for guests. # At the least, we need to load virtiofs here in order to mount /nix/store from the host. # This avoids the need to compile a custom kernel, which takes ages. @@ -125,7 +135,7 @@ let mktuntap = callPackage (import ./mktuntap) {}; - # Create a vs* script to launch a VM. + # Create a vs* script to launch a VM (s6-based rootfs). mkStart = { name, command ? "", cpus ? 2, network ? true, hostNumber, rootDev, disks, memory ? 4096 }: { inherit name; value = @@ -179,6 +189,58 @@ let ''; }; + # Create a vs* script to launch a NixOS-based VM (uses self-contained qcow2 image). + mkStartNixos = { name, command ? "", cpus ? 2, network ? true, hostNumber, disks, memory ? 4096 }: + { inherit name; + value = + '' + #!${execline}/bin/execlineb -s0 + foreground { umount /run/user/1000/doc } + backtick command { pipeline { echo -n $@ } base64 -w0 } + importas -iu command command + importas -iu XDG_RUNTIME_DIR XDG_RUNTIME_DIR + importas -iu WAYLAND_DISPLAY WAYLAND_DISPLAY + '' + lib.optionalString network '' + ${mktuntap}/bin/mktuntap -i tap${name} -pvB 3 + '' + '' + ${pkgs.s6}/bin/s6-softlimit -l 0 + "${lib.getBin crosvm}/bin/crosvm" run + -m ${toString memory} + --initrd=${rootfsNixos}/initrd + --serial=hardware=virtio-console + ${builtins.concatStringsSep "\n" disks} + --shared-dir "/home/david/vms/shared/${name}:mtdshared:type=fs" + -p "init=${rootfsNixos.config.system.build.toplevel}/init" + -p "net.ifnames=0" + -p "spectrumcmd=$command" + -p "spectrumname=${name}" + '' + + lib.optionalString network ( + let + ip = "10.0.0.${toString hostNumber}"; + gw = "10.0.0.${toString (hostNumber - 1)}"; + ipv6 = "fd4d:06ff:48e4:${toString (hostNumber - 1)}::2/48"; + gwv6 = "fd4d:06ff:48e4:${toString (hostNumber - 1)}::1"; + in + '' + -p "spectrumip=${ip}" + -p "spectrumgw=${gw}" + -p "spectrumip6=${ipv6}" + -p "spectrumgw6=${gwv6}" + --tap-fd 3 + '' + ) + '' + --cid ${toString hostNumber} + -p root=/dev/vda + --seccomp-log-failures + --cpus ${toString cpus} + --gpu=context-types=cross-domain:virgl2 + -s $\{XDG_RUNTIME_DIR}"/crosvm-${name}.sock" + --wayland-sock "$\{XDG_RUNTIME_DIR}/$\{WAYLAND_DISPLAY}" + "${rootfsNixos}/bzImage" + ''; + }; + # Configuration for each VM. vms = [ { @@ -246,10 +308,25 @@ let } ]; + # NixOS-based VMs (use self-contained qcow2 images with systemd) + nixosVms = [ + { + name = "fulltest"; + memory = 4096; + hostNumber = 15; + cpus = 4; + disks = [ + "--disk ${rootfsNixos}/nixos.qcow2" + ]; + } + ]; + # The list of vs* scripts. vmScripts = map mkStart vms; + nixosVmScripts = map mkStartNixos nixosVms; + allVmScripts = vmScripts ++ nixosVmScripts; - attrs = builtins.listToAttrs (map ({name, value}: { name = "${name}Script"; inherit value; }) vmScripts); + attrs = builtins.listToAttrs (map ({name, value}: { name = "${name}Script"; inherit value; }) allVmScripts); # Systemd unit file to also run the proxy on the host. This fixes some DPI problems with Xwayland. xwaylandService = '' @@ -293,12 +370,13 @@ in runCommand "qubes-lite" (attrs // { inherit xwaylandService; }) '' mkdir -p "$out/bin" - ${builtins.concatStringsSep "\n" (map ({name, value}: ''echo -n "$'' + ''${name}Script" > "$out/bin/vs${name}"'') vmScripts)} - ${builtins.concatStringsSep "\n" (map spawnApp vms)} + ${builtins.concatStringsSep "\n" (map ({name, value}: ''echo -n "$'' + ''${name}Script" > "$out/bin/vs${name}"'') allVmScripts)} + ${builtins.concatStringsSep "\n" (map spawnApp (vms ++ nixosVms))} chmod +x "$out/bin/"* ln -s "${wayland-proxy-virtwl}/bin/wayland-proxy-virtwl" "$out/bin/wayland-proxy-virtwl" mkdir -p "$out/share/systemd/user" echo -n "$xwaylandService" > "$out/share/systemd/user/xwayland-proxy.service" ln -s ${rootfs} "$out/rootfs-gc-keep" + ln -s ${rootfsNixos} "$out/rootfs-nixos-gc-keep" ln -s ${crosvm}/bin/crosvm "$out/bin/crosvm" '' diff --git a/flake.nix b/flake.nix index 5c5e95b..14cb3fa 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,8 @@ outputs = { self, nixpkgs, wayland-proxy-virtwl, crosvm }: let eachSystem = nixpkgs.lib.genAttrs ["x86_64-linux" "aarch64-linux"]; + + # Build the main qubes-lite package (s6-based rootfs) make = system: guestPrograms: let pkgs = nixpkgs.legacyPackages.${system}; in pkgs.callPackage (import ./default.nix) { @@ -21,12 +23,24 @@ wayland-proxy-virtwl = wayland-proxy-virtwl.packages.${system}.default; crosvm = crosvm.packages.${system}.default; }; + + # Build NixOS-based rootfs as qcow2 image + makeRootfsNixos = system: { guestPrograms ? [], guestConfig ? {} }: + let pkgs = nixpkgs.legacyPackages.${system}; in + pkgs.callPackage (import ./rootfs-nixos) { + inherit guestPrograms guestConfig; + wayland-proxy-virtwl = wayland-proxy-virtwl.packages.${system}.default; + }; in { packages = eachSystem(system: { default = make system []; + rootfs-nixos = makeRootfsNixos system {}; }); + # Helper function for building custom NixOS rootfs + lib.makeRootfsNixos = makeRootfsNixos; + nixosModules.default = { config, pkgs, lib, ... }: { options = { programs.qubes-lite.guestPrograms = lib.mkOption { diff --git a/rootfs-nixos/configuration.nix b/rootfs-nixos/configuration.nix new file mode 100644 index 0000000..1bf2b2d --- /dev/null +++ b/rootfs-nixos/configuration.nix @@ -0,0 +1,366 @@ +# NixOS configuration for qubes-lite guest VMs +# +# Features: +# - Systemd-based stage 1 initrd +# - Overlayfs root (read-only ext4 lower + tmpfs upper) +# - wayland-proxy-virtwl for GPU passthrough +# - Network configuration from kernel command line +# - vsock command listener for host communication +# +{ config, pkgs, lib, wayland-proxy-virtwl, ... }: + +{ + ############################################################################# + # Boot Configuration - Systemd Stage 1 + ############################################################################# + + boot.initrd.systemd.enable = true; + + # Kernel modules needed in initrd for virtio devices + boot.initrd.availableKernelModules = [ + "virtio_pci" + "virtio_blk" + "virtio_net" + "virtio_console" + "virtio_gpu" + "virtiofs" + "virtio_balloon" + "virtio_rng" + "ext4" + "overlay" + "vsock" + "vmw_vsock_virtio_transport" + "fuse" + ]; + + # Load these modules at boot + boot.initrd.kernelModules = [ "overlay" ]; + + # No bootloader - crosvm uses direct kernel boot + boot.loader.grub.enable = false; + + # Disable remount service - overlay filesystems cannot be remounted + systemd.services.systemd-remount-fs.serviceConfig.ExecStart = [ + "" # Clear the default ExecStart + "${pkgs.coreutils}/bin/true" # Do nothing successfully + ]; + + ############################################################################# + # Overlayfs Root Configuration + ############################################################################# + + # Dummy root filesystem entry - actual mount is handled by initrd + # We use "none" device to prevent NixOS from generating conflicting mounts + fileSystems."/" = { + device = "overlay"; + fsType = "overlay"; + options = [ "defaults" ]; + # Don't generate initrd mount - we handle it manually + neededForBoot = false; + }; + + # Custom sysroot.mount for overlay in initrd + boot.initrd.systemd.mounts = [{ + where = "/sysroot"; + what = "overlay"; + type = "overlay"; + options = "lowerdir=/sysroot/.overlay-lower,upperdir=/sysroot/.overlay-rw/upper,workdir=/sysroot/.overlay-rw/work"; + wantedBy = [ "initrd-root-fs.target" ]; + before = [ "initrd-root-fs.target" ]; + after = [ "prepare-overlay.service" ]; + requires = [ "prepare-overlay.service" ]; + unitConfig = { + DefaultDependencies = false; + RequiresMountsFor = "/sysroot/.overlay-lower /sysroot/.overlay-rw"; + }; + }]; + + # Prepare the overlay mounts before sysroot.mount + boot.initrd.systemd.services.prepare-overlay = { + description = "Prepare overlay filesystems"; + wantedBy = [ "initrd-root-fs.target" ]; + before = [ "sysroot.mount" ]; + after = [ "dev-vda.device" ]; + requires = [ "dev-vda.device" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + # Create mount points + mkdir -p /sysroot/.overlay-lower /sysroot/.overlay-rw + + # Mount ext4 as read-only lower layer + mount -t ext4 -o ro /dev/vda /sysroot/.overlay-lower + + # Mount tmpfs for writable upper layer + mount -t tmpfs -o mode=0755 tmpfs /sysroot/.overlay-rw + + # Create upper and work directories + mkdir -p /sysroot/.overlay-rw/upper /sysroot/.overlay-rw/work + ''; + }; + + ############################################################################# + # User Configuration + ############################################################################# + + boot.initrd.systemd.emergencyAccess = true; + + # Temp for testing + users.users.root.password = "foo"; + + users.users.user = { + isNormalUser = true; + uid = 1000; + group = "user"; + extraGroups = [ "wheel" "video" "audio" "input" ]; + # Home directory - user can configure persistent storage via guestConfig + home = "/home/user"; + createHome = true; + }; + + users.groups.user.gid = 1000; + + # Passwordless sudo for user + security.sudo.wheelNeedsPassword = false; + + ############################################################################# + # Network Configuration (from kernel parameters) + ############################################################################# + + networking.useDHCP = false; + networking.useNetworkd = true; + networking.usePredictableInterfaceNames = false; # Use eth0 instead of enp0s7 + + # Set hostname from kernel param spectrumname + systemd.services.set-hostname = { + description = "Set hostname from kernel command line"; + wantedBy = [ "multi-user.target" ]; + before = [ "network.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = pkgs.writeShellScript "set-hostname" '' + for param in $(cat /proc/cmdline); do + case "$param" in + spectrumname=*) + hostname="''${param#spectrumname=}" + ${pkgs.hostname}/bin/hostname "$hostname" + ;; + esac + done + ''; + }; + }; + + # Configure network from kernel parameters + systemd.services.configure-network = { + description = "Configure network from kernel command line"; + wantedBy = [ "multi-user.target" ]; + after = [ "systemd-networkd.service" "sys-subsystem-net-devices-eth0.device" ]; + wants = [ "sys-subsystem-net-devices-eth0.device" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = pkgs.writeShellScript "configure-network" '' + SPECTRUMIP="" + SPECTRUMGW="" + SPECTRUMIP6="" + SPECTRUMGW6="" + + for param in $(cat /proc/cmdline); do + case "$param" in + spectrumip=*) SPECTRUMIP="''${param#spectrumip=}" ;; + spectrumgw=*) SPECTRUMGW="''${param#spectrumgw=}" ;; + spectrumip6=*) SPECTRUMIP6="''${param#spectrumip6=}" ;; + spectrumgw6=*) SPECTRUMGW6="''${param#spectrumgw6=}" ;; + esac + done + + # Configure loopback + ${pkgs.iproute2}/bin/ip link set lo up + ${pkgs.iproute2}/bin/ip addr add 127.0.0.1/8 dev lo 2>/dev/null || true + + # Configure eth0 if IP provided + if [ -n "$SPECTRUMIP" ] && [ "$SPECTRUMIP" != "-" ]; then + ${pkgs.iproute2}/bin/ip addr add "$SPECTRUMIP/31" dev eth0 + ${pkgs.iproute2}/bin/ip link set eth0 up + ${pkgs.iproute2}/bin/ip route add default via "$SPECTRUMGW" + + if [ -n "$SPECTRUMIP6" ]; then + ${pkgs.iproute2}/bin/ip -6 addr add "$SPECTRUMIP6" dev eth0 + ${pkgs.iproute2}/bin/ip -6 route add default via "$SPECTRUMGW6" + fi + fi + ''; + }; + }; + + ############################################################################# + # wayland-proxy-virtwl Systemd Service + ############################################################################# + + systemd.services.wayland-proxy-virtwl = { + description = "Wayland proxy for virtio-gpu"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + path = [ pkgs.xwayland ]; + + environment = { + XDG_RUNTIME_DIR = "/run/user/1000"; + HOME = "/home/user"; + }; + + serviceConfig = { + Type = "simple"; + User = "user"; + Group = "user"; + + ExecStartPre = "+${pkgs.writeShellScript "setup-runtime-dir" '' + mkdir -p /run/user/1000 + chown user:user /run/user/1000 + chmod 700 /run/user/1000 + ''}"; + + ExecStart = pkgs.writeShellScript "start-wayland-proxy" '' + # Extract VM name from kernel command line + VMNAME="vm" + for param in $(cat /proc/cmdline); do + case "$param" in + spectrumname=*) VMNAME="''${param#spectrumname=}" ;; + esac + done + + exec ${wayland-proxy-virtwl}/bin/wayland-proxy-virtwl \ + --virtio-gpu \ + --tag="[$VMNAME] " \ + --x-display=0 \ + --xrdb Xft.dpi:150 \ + -v \ + --log-suppress motion \ + --log-ring-path /home/user/wayland.log \ + --x-unscale=2 \ + -- ${pkgs.dbus}/bin/dbus-launch --exit-with-session sleep infinity + ''; + + Restart = "on-failure"; + RestartSec = "5s"; + }; + }; + + ############################################################################# + # Command Execution via vsock + ############################################################################# + + # Listen for commands from host via vsock + systemd.services.vsock-command-listener = { + description = "Listen for commands from host via vsock"; + wantedBy = [ "multi-user.target" ]; + after = [ "wayland-proxy-virtwl.service" ]; + + environment = { + WAYLAND_DISPLAY = "wayland-0"; + DISPLAY = ":0"; + XDG_RUNTIME_DIR = "/run/user/1000"; + HOME = "/home/user"; + PATH = lib.mkForce "/run/current-system/sw/bin"; + }; + + serviceConfig = { + Type = "simple"; + User = "user"; + Group = "user"; + ExecStart = "${pkgs.socat}/bin/socat vsock-listen:5000,reuseaddr,fork exec:${pkgs.bashInteractive}/bin/bash"; + Restart = "always"; + RestartSec = "1s"; + }; + }; + + # Execute command from kernel param on boot + systemd.services.run-spectrum-cmd = { + description = "Execute command from kernel command line"; + wantedBy = [ "multi-user.target" ]; + after = [ "wayland-proxy-virtwl.service" ]; + + environment = { + WAYLAND_DISPLAY = "wayland-0"; + DISPLAY = ":0"; + XDG_RUNTIME_DIR = "/run/user/1000"; + HOME = "/home/user"; + }; + + path = [ pkgs.bashInteractive pkgs.coreutils ]; + + serviceConfig = { + Type = "oneshot"; + User = "user"; + Group = "user"; + ExecStart = pkgs.writeShellScript "run-spectrum-cmd" '' + SPECTRUMCMD="" + for param in $(cat /proc/cmdline); do + case "$param" in + spectrumcmd=*) SPECTRUMCMD="''${param#spectrumcmd=}" ;; + esac + done + + if [ -n "$SPECTRUMCMD" ]; then + echo "$SPECTRUMCMD" | base64 -d | ${pkgs.bashInteractive}/bin/bash + fi + ''; + }; + }; + + ############################################################################# + # Graphics & Hardware Configuration + ############################################################################# + + # OpenGL/Mesa for virtio-gpu + hardware.graphics.enable = true; + + # Device permissions for GPU and vsock + services.udev.extraRules = '' + KERNEL=="card0", SUBSYSTEM=="drm", MODE="0666" + KERNEL=="vsock", MODE="0666" + ''; + + ############################################################################# + # System Configuration + ############################################################################# + + # Locale and timezone + i18n.defaultLocale = "en_GB.UTF-8"; + time.timeZone = "Europe/London"; + + # Essential packages + environment.systemPackages = with pkgs; [ + bashInteractive + vim + socat + pciutils + util-linux + iproute2 + xwayland + xfce.xfce4-terminal + perf + util-linux + wl-clipboard + ]; + + # D-Bus for desktop applications + services.dbus.enable = true; + + # Fonts + fonts.packages = with pkgs; [ + source-code-pro + dejavu_fonts + ]; + + # Disable services we don't need + services.timesyncd.enable = false; + + # System version + system.stateVersion = "25.11"; +} diff --git a/rootfs-nixos/default.nix b/rootfs-nixos/default.nix new file mode 100644 index 0000000..2e6f849 --- /dev/null +++ b/rootfs-nixos/default.nix @@ -0,0 +1,74 @@ +# Build a NixOS system into a qcow2 image with overlayfs root +# +# Outputs: +# qcow2 - The disk image (raw ext4, no partitions) +# kernel - The kernel for direct boot +# initrd - The initramfs for direct boot +# +{ pkgs +, lib ? pkgs.lib +, wayland-proxy-virtwl +, guestPrograms ? [] +, guestConfig ? {} +}: + +let + # Evaluate the NixOS configuration + nixos = pkgs.nixos ({ config, pkgs, lib, ... }: { + imports = [ + ./configuration.nix + guestConfig + ]; + + # Pass wayland-proxy-virtwl to the configuration + _module.args.wayland-proxy-virtwl = wayland-proxy-virtwl; + + # Guest programs included directly in image + environment.systemPackages = guestPrograms; + }); + + # Empty directory for overlay mount points + emptyDir = pkgs.runCommand "empty-dir" {} "mkdir $out"; + + # Build the disk image + diskImage = import "${pkgs.path}/nixos/lib/make-disk-image.nix" { + inherit pkgs lib; + config = nixos.config; + + # Raw ext4 filesystem, no partition table + partitionTableType = "none"; + format = "qcow2"; + fsType = "ext4"; + label = "nixos"; + + # No bootloader - crosvm uses direct kernel boot + installBootLoader = false; + + # Size configuration + diskSize = "auto"; + additionalSpace = "512M"; + + # Don't include channel + copyChannel = false; + + # Create overlay mount point directories in the image + contents = [ + { source = emptyDir; target = "/.overlay-lower"; } + { source = emptyDir; target = "/.overlay-rw"; } + ]; + }; + + kernel = nixos.config.system.build.kernel; + initrd = nixos.config.system.build.initialRamdisk; + +in pkgs.runCommand "rootfs-nixos" { + passthru = { + inherit nixos diskImage kernel initrd; + config = nixos.config; + }; +} '' + mkdir -p $out + ln -s ${diskImage}/nixos.qcow2 $out/nixos.qcow2 + ln -s ${kernel}/bzImage $out/bzImage + ln -s ${initrd}/initrd $out/initrd +''