From e4c9e885a1786e2416efdfbc41c235b776978029 Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Thu, 6 Nov 2025 18:13:54 -0500 Subject: [PATCH 1/6] Add Darwin support via vfkit hypervisor Implements basic vfkit hypervisor runner to enable running Linux VMs on macOS using Apple's Virtualization.framework. vfkit is already in nixpkgs and is production-ready (used by minikube, podman, CRC). Features: - Kernel + initrd booting with proper console configuration - NAT networking (user mode) for basic connectivity - virtiofs shares for /nix/store (fast host directory sharing) - Volume support via virtio-blk - Graceful shutdown via Unix socket - Serial console support Limitations: - Darwin-only (requires macOS) - No bridge networking yet (NAT only; tap unavailable on macOS) - No device passthrough (Virtualization.framework limitation) - No vsock support yet (can be added later) The implementation follows existing hypervisor runner patterns and includes comprehensive feature validation with helpful error messages guiding users to supported alternatives. --- flake.nix | 84 ++++++++++++-------- lib/default.nix | 1 + lib/runners/vfkit.nix | 124 ++++++++++++++++++++++++++++++ nixos-modules/microvm/options.nix | 12 +++ 4 files changed, 189 insertions(+), 32 deletions(-) create mode 100644 lib/runners/vfkit.nix diff --git a/flake.nix b/flake.nix index 59fa0f79..433b8527 100644 --- a/flake.nix +++ b/flake.nix @@ -160,7 +160,8 @@ # currently broken: # "crosvm" ]; - hypervisorsWithUserNet = [ "qemu" "kvmtool" ]; + hypervisorsWithUserNet = [ "qemu" "kvmtool" "vfkit" ]; + hypervisorsDarwinOnly = [ "vfkit" ]; makeExample = { system, hypervisor, config ? {} }: lib.nixosSystem { system = lib.replaceString "-darwin" "-linux" system; @@ -176,12 +177,21 @@ nixpkgs.overlays = [ self.overlay ]; microvm = { inherit hypervisor; - # share the host's /nix/store if the hypervisor can do 9p - shares = lib.optional (builtins.elem hypervisor hypervisorsWith9p) { - tag = "ro-store"; - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - }; + # share the host's /nix/store if the hypervisor supports it + shares = + if builtins.elem hypervisor hypervisorsWith9p then [{ + tag = "ro-store"; + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + proto = "9p"; + }] + else if hypervisor == "vfkit" then [{ + tag = "ro-store"; + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + proto = "virtiofs"; + }] + else []; # writableStoreOverlay = "/nix/.rw-store"; # volumes = [ { # image = "nix-store-overlay.img"; @@ -212,34 +222,44 @@ }; in (builtins.foldl' (results: system: - builtins.foldl' ({ result, n }: hypervisor: { - result = result // { - "${system}-${hypervisor}-example" = makeExample { - inherit system hypervisor; - }; - } // - lib.optionalAttrs (builtins.elem hypervisor self.lib.hypervisorsWithNetwork) { - "${system}-${hypervisor}-example-with-tap" = makeExample { - inherit system hypervisor; - config = _: { - microvm.interfaces = [ { - type = "tap"; - id = "vm-${builtins.substring 0 4 hypervisor}"; - mac = "02:00:00:01:01:0${toString n}"; - } ]; - networking = { - interfaces.eth0.useDHCP = true; - firewall.allowedTCPPorts = [ 22 ]; - }; - services.openssh = { - enable = true; - settings.PermitRootLogin = "yes"; + builtins.foldl' ({ result, n }: hypervisor: + let + # Skip darwin-only hypervisors on Linux systems + isDarwinOnly = builtins.elem hypervisor hypervisorsDarwinOnly; + isDarwinSystem = lib.hasSuffix "-darwin" system; + shouldSkip = isDarwinOnly && !isDarwinSystem; + in + if shouldSkip then { inherit result n; } + else { + result = result // { + "${system}-${hypervisor}-example" = makeExample { + inherit system hypervisor; + }; + } // + # Skip tap example for darwin-only hypervisors (vfkit doesn't support tap) + lib.optionalAttrs (builtins.elem hypervisor self.lib.hypervisorsWithNetwork && !isDarwinOnly) { + "${system}-${hypervisor}-example-with-tap" = makeExample { + inherit system hypervisor; + config = _: { + microvm.interfaces = [ { + type = "tap"; + id = "vm-${builtins.substring 0 4 hypervisor}"; + mac = "02:00:00:01:01:0${toString n}"; + } ]; + networking = { + interfaces.eth0.useDHCP = true; + firewall.allowedTCPPorts = [ 22 ]; + }; + services.openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; }; }; }; - }; - n = n + 1; - }) results self.lib.hypervisors + n = n + 1; + } + ) results self.lib.hypervisors ) { result = {}; n = 1; } systems).result; }; } diff --git a/lib/default.nix b/lib/default.nix index 25223679..87b7fa29 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -8,6 +8,7 @@ rec { "kvmtool" "stratovirt" "alioth" + "vfkit" ]; hypervisorsWithNetwork = hypervisors; diff --git a/lib/runners/vfkit.nix b/lib/runners/vfkit.nix new file mode 100644 index 00000000..4e92c207 --- /dev/null +++ b/lib/runners/vfkit.nix @@ -0,0 +1,124 @@ +{ pkgs +, microvmConfig +, macvtapFds +, withDriveLetters +, ... +}: + +let + inherit (pkgs) lib; + inherit (vmHostPackages.stdenv.hostPlatform) system; + inherit (microvmConfig) vmHostPackages; + + vfkit = vmHostPackages.vfkit; + + inherit (microvmConfig) + hostName vcpu mem user interfaces volumes shares socket + storeOnDisk kernel initrdPath storeDisk kernelParams + balloon devices credentialFiles vsock; + + inherit (microvmConfig.vfkit) extraArgs logLevel; + + volumesWithLetters = withDriveLetters microvmConfig; + + # vfkit requires uncompressed kernel + kernelPath = "${kernel.out}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"; + + kernelCmdLine = "console=hvc0 reboot=t panic=-1 ${toString kernelParams}"; + + bootloaderArgs = [ + "--bootloader" + "linux,kernel=${kernelPath},initrd=${initrdPath},cmdline=\"${builtins.concatStringsSep " " kernelCmdLine}\"" + ]; + + deviceArgs = + [ "--device" "virtio-rng" ] + ++ + [ "--device" "virtio-serial,stdio" ] + ++ + (builtins.concatMap ({ image, ... }: [ + "--device" "virtio-blk,path=${image}" + ]) volumesWithLetters) + ++ (builtins.concatMap ({ proto, source, tag, ... }: + if proto == "virtiofs" then [ + "--device" "virtio-fs,sharedDir=${source},mountTag=${tag}" + ] + else + throw "vfkit does not support ${proto} share. Use proto = \"virtiofs\" instead." + ) shares) + ++ (builtins.concatMap ({ type, id, mac, ... }: + if type == "user" then [ + "--device" "virtio-net,nat,mac=${mac}" + ] + else if type == "bridge" then + throw "vfkit bridge networking requires vmnet-helper which is not yet implemented. Use type = \"user\" for NAT networking." + else + throw "Unknown network interface type: ${type}" + ) interfaces); + + allArgsWithoutSocket = [ + "${lib.getExe vfkit}" + "--cpus" (toString vcpu) + "--memory" (toString mem) + ] + ++ lib.optionals (logLevel != null) [ + "--log-level" logLevel + ] + ++ bootloaderArgs + ++ deviceArgs + ++ extraArgs; + +in +{ + tapMultiQueue = false; + + preStart = lib.optionalString (socket != null) '' + rm -f ${socket} + ''; + + command = + if !vmHostPackages.stdenv.hostPlatform.isDarwin + then throw "vfkit only works on macOS (Darwin). Current host: ${system}" + else if vmHostPackages.stdenv.hostPlatform.isAarch64 != pkgs.stdenv.hostPlatform.isAarch64 + then throw "vfkit requires matching host and guest architectures. Host: ${system}, Guest: ${pkgs.stdenv.hostPlatform.system}" + else if user != null + then throw "vfkit does not support changing user" + else if balloon + then throw "vfkit does not support memory ballooning" + else if devices != [] + then throw "vfkit does not support device passthrough" + else if credentialFiles != {} + then throw "vfkit does not support credentialFiles" + else if vsock.cid != null + then throw "vfkit vsock support not yet implemented in microvm.nix" + else if storeOnDisk + then throw "vfkit does not support storeOnDisk. Use virtiofs shares instead (already configured in examples)." + else + let + baseCmd = lib.escapeShellArgs allArgsWithoutSocket; + vfkitCmd = lib.concatStringsSep " " (map lib.escapeShellArg allArgsWithoutSocket); + in + # vfkit requires absolute socket paths, so expand relative paths + if socket != null + then "bash -c ${lib.escapeShellArg '' + SOCKET_ABS=${lib.escapeShellArg socket} + [[ "$SOCKET_ABS" != /* ]] && SOCKET_ABS="$PWD/$SOCKET_ABS" + exec ${vfkitCmd} --restful-uri "unix:///$SOCKET_ABS" + ''}" + else baseCmd; + + canShutdown = socket != null; + + shutdownCommand = + if socket != null + then '' + SOCKET_ABS="${lib.escapeShellArg socket}" + [[ "$SOCKET_ABS" != /* ]] && SOCKET_ABS="$PWD/$SOCKET_ABS" + echo '{"state": "Stop"}' | ${vmHostPackages.socat}/bin/socat - "UNIX-CONNECT:$SOCKET_ABS" + '' + else throw "Cannot shutdown without socket"; + + supportsNotifySocket = false; + + requiresMacvtapAsFds = false; +} diff --git a/nixos-modules/microvm/options.nix b/nixos-modules/microvm/options.nix index a241d30b..b5d623be 100644 --- a/nixos-modules/microvm/options.nix +++ b/nixos-modules/microvm/options.nix @@ -596,6 +596,18 @@ in description = "Type of IO engine to use for Firecracker drives (disks)."; }; + vfkit.extraArgs = mkOption { + type = with types; listOf str; + default = []; + description = "Extra arguments to pass to vfkit."; + }; + + vfkit.logLevel = mkOption { + type = with types; nullOr (enum ["debug" "info" "error"]); + default = "info"; + description = "vfkit log level."; + }; + prettyProcnames = mkOption { type = types.bool; default = true; From ebe491ed6581e2aeef86e72e7156e65c38a9fb6d Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Fri, 7 Nov 2025 11:32:47 -0500 Subject: [PATCH 2/6] docs: Update documentation for vfkit hypervisor support Updates all documentation to reflect the addition of vfkit as a new hypervisor option for running MicroVMs on macOS. Changes: - Add vfkit to hypervisor comparison table with its restrictions - Update intro and README to mention macOS support - Document vfkit's user-mode (NAT) networking support - Note that vfkit has built-in virtiofs (no separate virtiofsd needed) - Add vfkit-specific configuration options to options table - Include vfkit-example in the examples section --- README.md | 30 +++++++++++++++++------------- doc/src/interfaces.md | 5 ++++- doc/src/intro.md | 7 ++++--- doc/src/options.md | 2 ++ doc/src/shares.md | 5 ++++- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 74ee0a61..4c1357b8 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@

A Nix Flake to build NixOS and run it on one of several Type-2 -Hypervisors on NixOS/Linux. The project is intended to provide a more +Hypervisors on NixOS/Linux or macOS. The project is intended to provide a more isolated alternative to `nixos-container`. You can either build and run MicroVMs like Nix packages, or alternatively install them as systemd services declaratively in your host's Nix Flake or @@ -26,8 +26,8 @@ imperatively with the provided `microvm` command. - MicroVMs are Virtual Machines but use special device interfaces (virtio) for high performance. -- This project runs them on NixOS hosts. -- You can choose one of five hypervisors for each MicroVM. +- This project runs them on NixOS/Linux hosts and macOS (via vfkit). +- You can choose from eight hypervisors for each MicroVM. - MicroVMs have a fixed RAM allocation (default: 512 MB) but can be shrunk using `microvm-balloon` - MicroVMs have a read-only root disk with either a prepopulated @@ -40,20 +40,21 @@ imperatively with the provided `microvm` command. a block device, or alternatively as a shared directory hierarchy through *9p* or *virtiofs*. - Zero, one, or more virtual tap ethernet network interfaces can be - attached to a MicroVM. `qemu` and `kvmtool` also support *user* + attached to a MicroVM. `qemu`, `kvmtool`, and `vfkit` also support *user* networking which requires no additional setup on the host. ## Hypervisors -| Hypervisor | Language | Restrictions | -|-------------------------------------------------------------------------|----------|------------------------------------------| -| [qemu](https://www.qemu.org/) | C | | -| [cloud-hypervisor](https://www.cloudhypervisor.org/) | Rust | no 9p shares | -| [firecracker](https://firecracker-microvm.github.io/) | Rust | no 9p/virtiofs shares | -| [crosvm](https://chromium.googlesource.com/chromiumos/platform/crosvm/) | Rust | 9p shares broken | -| [kvmtool](https://github.com/kvmtool/kvmtool) | C | no virtiofs shares, no control socket | -| [stratovirt](https://github.com/openeuler-mirror/stratovirt) | Rust | no 9p/virtiofs shares, no control socket | -| [alioth](https://github.com/google/alioth) | Rust | no virtiofs shares, no control socket | +| Hypervisor | Language | Restrictions | +|-------------------------------------------------------------------------|----------|-------------------------------------------------------| +| [qemu](https://www.qemu.org/) | C | | +| [cloud-hypervisor](https://www.cloudhypervisor.org/) | Rust | no 9p shares | +| [firecracker](https://firecracker-microvm.github.io/) | Rust | no 9p/virtiofs shares | +| [crosvm](https://chromium.googlesource.com/chromiumos/platform/crosvm/) | Rust | 9p shares broken | +| [kvmtool](https://github.com/kvmtool/kvmtool) | C | no virtiofs shares, no control socket | +| [stratovirt](https://github.com/openeuler-mirror/stratovirt) | Rust | no 9p/virtiofs shares, no control socket | +| [alioth](https://github.com/google/alioth) | Rust | no virtiofs shares, no control socket | +| [vfkit](https://github.com/crc-org/vfkit) | Go | macOS only, no 9p shares, no tap/bridge networking | ## Installation @@ -85,6 +86,9 @@ nix run microvm#cloud-hypervisor-example nix run microvm#crosvm-example nix run microvm#kvmtool-example nix run microvm#stratovirt-example + +# On macOS only: +nix run microvm#vfkit-example ``` ### Run a MicroVM example with nested MicroVMs on 5 different Hypervisors diff --git a/doc/src/interfaces.md b/doc/src/interfaces.md index 9c494abd..73b85a9d 100644 --- a/doc/src/interfaces.md +++ b/doc/src/interfaces.md @@ -20,12 +20,15 @@ configuration: ## `type = "user"` -User-mode networking is only provided by qemu and kvmtool, providing +User-mode networking is provided by qemu, kvmtool, and vfkit, providing outgoing connectivity to your MicroVM without any further setup. As kvmtool seems to lack a built-in DHCP server, additional static IP configuration is necessary inside the MicroVM. +**Note:** vfkit (macOS) only supports user-mode networking. TAP and bridge +networking are not available. + ## `type = "tap"` Use a virtual tuntap Ethernet interface. Its name is the value of diff --git a/doc/src/intro.md b/doc/src/intro.md index dbce17d6..f4886c63 100644 --- a/doc/src/intro.md +++ b/doc/src/intro.md @@ -1,9 +1,9 @@ # Intro **microvm.nix** is a Flake to run lightweight NixOS virtual machines -on NixOS. Starting with the reasons why for the remainder of this +on NixOS and macOS. Starting with the reasons why for the remainder of this chapter, this handbook guides you through the provisioning of MicroVMs -on your NixOS machine. +on your NixOS or macOS machine. ## Compartmentalization @@ -47,4 +47,5 @@ overhead has been reduced a lot by replacing emulated devices with This Flake offers you to run your MicroVMs not only on QEMU but with other Hypervisors that have been explicitly authored for *virtio*. Some of them are written in Rust, a programming language -that is renowned for being safer than C. +that is renowned for being safer than C. On macOS, vfkit leverages +Apple's native Virtualization.framework for running Linux VMs. diff --git a/doc/src/options.md b/doc/src/options.md index 71b5852f..54a346f0 100644 --- a/doc/src/options.md +++ b/doc/src/options.md @@ -15,6 +15,8 @@ available for customization. These are the most important ones: | `microvm.socket` | Control socket for the Hypervisor so that a MicroVM can be shutdown cleanly | | `microvm.user` | (qemu only) User account which Qemu will switch to when started as root | | `microvm.forwardPorts` | (qemu user-networking only) TCP/UDP port forwarding | +| `microvm.vfkit.extraArgs` | (vfkit only) Extra arguments to pass to vfkit | +| `microvm.vfkit.logLevel` | (vfkit only) Log level: "debug", "info", or "error" (default: "info") | | `microvm.kernelParams` | Like `boot.kernelParams` but will not end up in `system.build.toplevel`, saving you rebuilds | | `microvm.storeOnDisk` | Enables the store on the boot squashfs even in the presence of a share with the host's `/nix/store` | | `microvm.writableStoreOverlay` | Optional string of the path where all writes to `/nix/store` should go to. | diff --git a/doc/src/shares.md b/doc/src/shares.md index 290d71d6..53a60122 100644 --- a/doc/src/shares.md +++ b/doc/src/shares.md @@ -10,7 +10,7 @@ In `microvm.shares` elements the `proto` field allows either of two values: - `9p` (default) is built into many hypervisors, allowing you to - quickly share a directory tree + quickly share a directory tree. Not supported by vfkit on macOS. - `virtiofs` requires a separate virtiofsd service which is started as a prerequisite when you start MicroVMs through a systemd service @@ -21,6 +21,9 @@ values: Expect `virtiofs` to yield better performance over `9p`. + **Note:** vfkit (macOS) has built-in virtiofs support and does not + require a separate virtiofsd service. + ```nix microvm.shares = [ { proto = "virtiofs"; From e21c05abdec8afb0ed9ce56dff326039505f7c3b Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Fri, 7 Nov 2025 11:44:11 -0500 Subject: [PATCH 3/6] feat: Add graphics support for vfkit hypervisor Enables vfkit to use the --gui flag and virtio-gpu devices when microvm.graphics.enable is set to true, matching the behavior of other hypervisors like qemu and cloud-hypervisor. Changes: - Add virtio-gpu, virtio-input keyboard and pointing devices when graphics.enable is true - Add --gui flag to vfkit command line when graphics are enabled - Replace virtio-serial with GUI devices in graphics mode - Update README to document graphics support on macOS with vfkit --- README.md | 5 ++++- lib/runners/vfkit.nix | 25 ++++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4c1357b8..bc874178 100644 --- a/README.md +++ b/README.md @@ -100,12 +100,15 @@ nix run microvm#vm Check `networkctl status virbr0` for the DHCP leases of the nested MicroVMs. They listen for ssh with an empty root password. -### Experimental: run graphical applications in cloud-hypervisor with Wayland forwarding +### Experimental: run graphical applications with graphics support +On Linux with cloud-hypervisor and Wayland forwarding: ```shell nix run microvm#graphics neverball ``` +On macOS with vfkit, enable graphics with `microvm.graphics.enable = true`. + ## Commercial support Accelerate your operations and secure your infrastructure with support from a diff --git a/lib/runners/vfkit.nix b/lib/runners/vfkit.nix index 4e92c207..34ecd659 100644 --- a/lib/runners/vfkit.nix +++ b/lib/runners/vfkit.nix @@ -15,7 +15,7 @@ let inherit (microvmConfig) hostName vcpu mem user interfaces volumes shares socket storeOnDisk kernel initrdPath storeDisk kernelParams - balloon devices credentialFiles vsock; + balloon devices credentialFiles vsock graphics; inherit (microvmConfig.vfkit) extraArgs logLevel; @@ -24,7 +24,9 @@ let # vfkit requires uncompressed kernel kernelPath = "${kernel.out}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"; - kernelCmdLine = "console=hvc0 reboot=t panic=-1 ${toString kernelParams}"; + kernelConsole = if graphics.enable then "tty0" else "hvc0"; + + kernelCmdLine = [ "console=${kernelConsole}" "reboot=t" "panic=-1" ] ++ kernelParams; bootloaderArgs = [ "--bootloader" @@ -32,11 +34,17 @@ let ]; deviceArgs = - [ "--device" "virtio-rng" ] - ++ - [ "--device" "virtio-serial,stdio" ] - ++ - (builtins.concatMap ({ image, ... }: [ + [ + "--device" "virtio-rng" + ] + ++ (if graphics.enable then [ + "--device" "virtio-gpu" + "--device" "virtio-input,keyboard" + "--device" "virtio-input,pointing" + ] else [ + "--device" "virtio-serial,stdio" + ]) + ++ (builtins.concatMap ({ image, ... }: [ "--device" "virtio-blk,path=${image}" ]) volumesWithLetters) ++ (builtins.concatMap ({ proto, source, tag, ... }: @@ -64,6 +72,9 @@ let ++ lib.optionals (logLevel != null) [ "--log-level" logLevel ] + ++ lib.optionals graphics.enable [ + "--gui" + ] ++ bootloaderArgs ++ deviceArgs ++ extraArgs; From 21185d0b7dea3204f091b5dff7e7345c280d96fe Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Fri, 7 Nov 2025 14:03:52 -0500 Subject: [PATCH 4/6] feat: Add Rosetta support for vfkit on Apple Silicon Enables running x86_64 (Intel) binaries in ARM64 Linux VMs on Apple Silicon Macs using Apple's Rosetta translation layer. Changes: - Add microvm.vfkit.rosetta.enable option to enable Rosetta - Add microvm.vfkit.rosetta.install option to auto-install Rosetta - Add microvm.vfkit.rosetta.ignoreIfMissing for compatibility - Add rosetta device to vfkit runner when enabled - Add NixOS module to automatically configure guest (mount + binfmt) - Validate that Rosetta is only used on aarch64-darwin systems - Create comprehensive documentation with setup instructions --- doc/src/SUMMARY.md | 1 + doc/src/options.md | 1 + doc/src/vfkit-rosetta.md | 74 +++++++++++++++++++++++++++++++ lib/runners/vfkit.nix | 18 ++++++-- nixos-modules/microvm/default.nix | 1 + nixos-modules/microvm/options.nix | 32 +++++++++++++ nixos-modules/microvm/rosetta.nix | 19 ++++++++ 7 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 doc/src/vfkit-rosetta.md create mode 100644 nixos-modules/microvm/rosetta.nix diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 4f9e1175..18d89968 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -8,6 +8,7 @@ - [Device pass-through](./devices.md) - [CPU emulation](./cpu-emulation.md) - [Output options](./output-options.md) + - [Using Rosetta with vfkit (macOS)](./vfkit-rosetta.md) - [MicroVM options reference ⚙️](./microvm-options.md) - [Running a MicroVM as a package](./packages.md) - [Preparing a host for declarative MicroVMs](./host.md) diff --git a/doc/src/options.md b/doc/src/options.md index 54a346f0..308779f9 100644 --- a/doc/src/options.md +++ b/doc/src/options.md @@ -17,6 +17,7 @@ available for customization. These are the most important ones: | `microvm.forwardPorts` | (qemu user-networking only) TCP/UDP port forwarding | | `microvm.vfkit.extraArgs` | (vfkit only) Extra arguments to pass to vfkit | | `microvm.vfkit.logLevel` | (vfkit only) Log level: "debug", "info", or "error" (default: "info") | +| `microvm.vfkit.rosetta.enable` | (vfkit only) Enable Rosetta for running x86_64 binaries on ARM64 (Apple Silicon only) | | `microvm.kernelParams` | Like `boot.kernelParams` but will not end up in `system.build.toplevel`, saving you rebuilds | | `microvm.storeOnDisk` | Enables the store on the boot squashfs even in the presence of a share with the host's `/nix/store` | | `microvm.writableStoreOverlay` | Optional string of the path where all writes to `/nix/store` should go to. | diff --git a/doc/src/vfkit-rosetta.md b/doc/src/vfkit-rosetta.md new file mode 100644 index 00000000..e15fc017 --- /dev/null +++ b/doc/src/vfkit-rosetta.md @@ -0,0 +1,74 @@ +# Using Rosetta with vfkit on Apple Silicon + +Rosetta support enables running x86_64 (Intel) binaries in your ARM64 Linux VM on Apple Silicon Macs. This is useful for running legacy applications or development tools that haven't been ported to ARM yet. + +## Requirements + +- Apple Silicon (M1/M2/M3/etc.) Mac +- macOS with Rosetta installed +- vfkit hypervisor + +## Configuration + +Enable Rosetta in your MicroVM configuration: + +```nix +{ + microvm = { + hypervisor = "vfkit"; + + vfkit.rosetta = { + enable = true; + # Optional: install Rosetta automatically if missing + install = true; + }; + }; +} +``` + +The NixOS module automatically handles mounting the Rosetta virtiofs share and configuring binfmt to use Rosetta for x86_64 binaries. No additional guest configuration is required. + +## Usage + +Once configured, you can run any x86_64 binary in your ARM64 VM. To verify Rosetta is working: + +```nix +# Add an x86_64 package to your configuration +environment.systemPackages = with pkgs; [ + file # to verify binary architecture + (pkgsCross.gnu64.hello) # x86_64 version of hello +]; +``` + +Then in the VM: + +```bash +# Verify you're running on ARM64 +uname -m +# Output: aarch64 + +# Check the binary architecture +file $(which hello) +# Output: ELF 64-bit LSB executable, x86-64, ... + +# Run the x86_64 binary via Rosetta +hello +# Output: Hello, world! +``` + +You can use `pkgsCross.gnu64.` to cross-compile any package from nixpkgs to x86_64 and run it via Rosetta. + +## Options Reference + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `microvm.vfkit.rosetta.enable` | bool | `false` | Enable Rosetta support | +| `microvm.vfkit.rosetta.install` | bool | `false` | Auto-install Rosetta if missing | +| `microvm.vfkit.rosetta.ignoreIfMissing` | bool | `false` | Continue if Rosetta unavailable | + +## Limitations + +- Only works on Apple Silicon Macs (M-series chips) +- vfkit will fail to start on Intel Macs if Rosetta is enabled +- Performance is slower than native ARM64 execution +- Not all x86_64 binaries may work perfectly diff --git a/lib/runners/vfkit.nix b/lib/runners/vfkit.nix index 34ecd659..2ad6b630 100644 --- a/lib/runners/vfkit.nix +++ b/lib/runners/vfkit.nix @@ -17,7 +17,7 @@ let storeOnDisk kernel initrdPath storeDisk kernelParams balloon devices credentialFiles vsock graphics; - inherit (microvmConfig.vfkit) extraArgs logLevel; + inherit (microvmConfig.vfkit) extraArgs logLevel rosetta; volumesWithLetters = withDriveLetters microvmConfig; @@ -61,8 +61,18 @@ let else if type == "bridge" then throw "vfkit bridge networking requires vmnet-helper which is not yet implemented. Use type = \"user\" for NAT networking." else - throw "Unknown network interface type: ${type}" - ) interfaces); + throw "vfkit does not support ${type} networking on macOS. Use type = \"user\" for NAT networking." + ) interfaces) + ++ lib.optionals rosetta.enable ( + let + rosettaArgs = builtins.concatStringsSep "," ( + [ "rosetta" "mountTag=rosetta" ] + ++ lib.optional rosetta.install "install" + ++ lib.optional rosetta.ignoreIfMissing "ignoreIfMissing" + ); + in + [ "--device" rosettaArgs ] + ); allArgsWithoutSocket = [ "${lib.getExe vfkit}" @@ -96,6 +106,8 @@ in then throw "vfkit does not support changing user" else if balloon then throw "vfkit does not support memory ballooning" + else if rosetta.enable && !vmHostPackages.stdenv.hostPlatform.isAarch64 + then throw "Rosetta requires Apple Silicon (aarch64-darwin). Current host: ${system}" else if devices != [] then throw "vfkit does not support device passthrough" else if credentialFiles != {} diff --git a/nixos-modules/microvm/default.nix b/nixos-modules/microvm/default.nix index 4752fdf4..7fa27df5 100644 --- a/nixos-modules/microvm/default.nix +++ b/nixos-modules/microvm/default.nix @@ -19,6 +19,7 @@ in ./pci-devices.nix ./virtiofsd ./graphics.nix + ./rosetta.nix ./optimization.nix ./ssh-deploy.nix ]; diff --git a/nixos-modules/microvm/options.nix b/nixos-modules/microvm/options.nix index b5d623be..9aed13eb 100644 --- a/nixos-modules/microvm/options.nix +++ b/nixos-modules/microvm/options.nix @@ -608,6 +608,38 @@ in description = "vfkit log level."; }; + vfkit.rosetta = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enable Rosetta support for running x86_64 binaries in ARM64 Linux VMs. + Only works on Apple Silicon (ARM) Macs. + + When enabled, the Rosetta virtiofs share will be automatically mounted + and binfmt will be configured to use Rosetta for x86_64 binaries. + ''; + }; + + install = mkOption { + type = types.bool; + default = false; + description = '' + Automatically install Rosetta if missing. + If false and Rosetta is not installed, vfkit will fail to start. + ''; + }; + + ignoreIfMissing = mkOption { + type = types.bool; + default = false; + description = '' + Continue execution even if Rosetta installation fails or is unavailable. + Useful for configurations that should work on both ARM and Intel Macs. + ''; + }; + }; + prettyProcnames = mkOption { type = types.bool; default = true; diff --git a/nixos-modules/microvm/rosetta.nix b/nixos-modules/microvm/rosetta.nix new file mode 100644 index 00000000..2d5f5be6 --- /dev/null +++ b/nixos-modules/microvm/rosetta.nix @@ -0,0 +1,19 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.microvm.vfkit.rosetta; + mountPoint = "/run/rosetta"; + mountTag = "rosetta"; +in +lib.mkIf (config.microvm.hypervisor == "vfkit" && cfg.enable) { + fileSystems.${mountPoint} = { + device = mountTag; + fsType = "virtiofs"; + }; + + boot.binfmt.registrations.rosetta = { + interpreter = "${mountPoint}/rosetta"; + magicOrExtension = ''\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00''; + mask = ''\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff''; + }; +} From 6904c5628831b6c45057a48a1cd42a56949d83b5 Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Fri, 7 Nov 2025 19:22:09 -0500 Subject: [PATCH 5/6] refactor: Simplify flake.nix example generation logic Replace complex nested foldl' blocks with clearer declarative code: - Extract helper functions (isDarwinOnly, isDarwinSystem, hypervisorSupportsSystem) to make platform checks readable - Replace accumulation pattern with filter + map approach - Use shouldInclude field on examples for consistent filtering - Simplify packages wrapping with concatMap No functional changes - all 58 configurations preserved exactly. Validated with: nix eval .#nixosConfigurations --apply builtins.attrNames --- flake.nix | 115 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 49 deletions(-) diff --git a/flake.nix b/flake.nix index 433b8527..dbfec91b 100644 --- a/flake.nix +++ b/flake.nix @@ -108,18 +108,23 @@ }; } // # wrap self.nixosConfigurations in executable packages - builtins.foldl' (result: systemName: - let - nixos = self.nixosConfigurations.${systemName}; - name = builtins.replaceStrings [ "${system}-" ] [ "" ] systemName; - inherit (nixos.config.microvm) hypervisor; - in - if nixos.pkgs.stdenv.hostPlatform.system == lib.replaceString "-darwin" "-linux" system - then result // { - "${name}" = nixos.config.microvm.runner.${hypervisor}; + lib.listToAttrs ( + lib.concatMap (configName: + let + config = self.nixosConfigurations.${configName}; + packageName = lib.replaceString "${system}-" "" configName; + # Check if this config's guest system matches our current build system + # (accounting for darwin hosts building linux guests) + guestSystem = config.pkgs.stdenv.hostPlatform.system; + buildSystem = lib.replaceString "-darwin" "-linux" system; + in + lib.optional (guestSystem == buildSystem) + { + name = packageName; + value = config.config.microvm.runner.${config.config.microvm.hypervisor}; } - else result - ) {} (builtins.attrNames self.nixosConfigurations); + ) (builtins.attrNames self.nixosConfigurations) + ); # Takes too much memory in `nix flake show` # checks = import ./checks { inherit self nixpkgs system; }; @@ -162,6 +167,17 @@ ]; hypervisorsWithUserNet = [ "qemu" "kvmtool" "vfkit" ]; hypervisorsDarwinOnly = [ "vfkit" ]; + hypervisorsWithTap = builtins.filter + # vfkit supports networking, but does not support tap + (hv: hv != "vfkit") + self.lib.hypervisorsWithNetwork; + + isDarwinOnly = hypervisor: builtins.elem hypervisor hypervisorsDarwinOnly; + isDarwinSystem = system: lib.hasSuffix "-darwin" system; + hypervisorSupportsSystem = hypervisor: system: + # Darwin-only hypervisors only work on darwin, others work everywhere + !(isDarwinOnly hypervisor && !(isDarwinSystem system)); + makeExample = { system, hypervisor, config ? {} }: lib.nixosSystem { system = lib.replaceString "-darwin" "-linux" system; @@ -220,46 +236,47 @@ config ]; }; - in - (builtins.foldl' (results: system: - builtins.foldl' ({ result, n }: hypervisor: - let - # Skip darwin-only hypervisors on Linux systems - isDarwinOnly = builtins.elem hypervisor hypervisorsDarwinOnly; - isDarwinSystem = lib.hasSuffix "-darwin" system; - shouldSkip = isDarwinOnly && !isDarwinSystem; - in - if shouldSkip then { inherit result n; } - else { - result = result // { - "${system}-${hypervisor}-example" = makeExample { - inherit system hypervisor; - }; - } // - # Skip tap example for darwin-only hypervisors (vfkit doesn't support tap) - lib.optionalAttrs (builtins.elem hypervisor self.lib.hypervisorsWithNetwork && !isDarwinOnly) { - "${system}-${hypervisor}-example-with-tap" = makeExample { - inherit system hypervisor; - config = _: { - microvm.interfaces = [ { - type = "tap"; - id = "vm-${builtins.substring 0 4 hypervisor}"; - mac = "02:00:00:01:01:0${toString n}"; - } ]; - networking = { - interfaces.eth0.useDHCP = true; - firewall.allowedTCPPorts = [ 22 ]; - }; - services.openssh = { - enable = true; - settings.PermitRootLogin = "yes"; - }; + + basicExamples = lib.flatten ( + lib.map (system: + lib.map (hypervisor: { + name = "${system}-${hypervisor}-example"; + value = makeExample { inherit system hypervisor; }; + shouldInclude = hypervisorSupportsSystem hypervisor system; + }) self.lib.hypervisors + ) systems + ); + + tapExamples = lib.flatten ( + lib.map (system: + lib.imap1 (idx: hypervisor: { + name = "${system}-${hypervisor}-example-with-tap"; + value = makeExample { + inherit system hypervisor; + config = _: { + microvm.interfaces = [ { + type = "tap"; + id = "vm-${builtins.substring 0 4 hypervisor}"; + mac = "02:00:00:01:01:0${toString idx}"; + } ]; + networking = { + interfaces.eth0.useDHCP = true; + firewall.allowedTCPPorts = [ 22 ]; + }; + services.openssh = { + enable = true; + settings.PermitRootLogin = "yes"; }; }; }; - n = n + 1; - } - ) results self.lib.hypervisors - ) { result = {}; n = 1; } systems).result; + shouldInclude = builtins.elem hypervisor hypervisorsWithTap + && hypervisorSupportsSystem hypervisor system; + }) self.lib.hypervisors + ) systems + ); + + included = builtins.filter (ex: ex.shouldInclude) (basicExamples ++ tapExamples); + in + builtins.listToAttrs (builtins.map ({ name, value, ... }: { inherit name value; }) included); }; } From c62a314675501bca658158b660344b41e2989205 Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Fri, 7 Nov 2025 19:24:14 -0500 Subject: [PATCH 6/6] fix: Only create darwin examples for hypervisors that work on macOS Most hypervisors (firecracker, cloud-hypervisor, crosvm, kvmtool, stratovirt, alioth) require Linux KVM and cannot run on macOS. Only qemu (via Apple's HVF) and vfkit work on darwin, so only create example configurations for these two hypervisors on darwin systems. This removes 24 misleading example configurations that could be built but never run on macOS: - aarch64-darwin-{alioth,cloud-hypervisor,crosvm,firecracker,kvmtool,stratovirt}-* - x86_64-darwin-{alioth,cloud-hypervisor,crosvm,firecracker,kvmtool,stratovirt}-* Net result: - Added: 2 vfkit examples (aarch64-darwin, x86_64-darwin) - Removed: 24 KVM-only darwin examples - Kept: All Linux examples + qemu darwin examples (HVF) --- flake.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index dbfec91b..58829f82 100644 --- a/flake.nix +++ b/flake.nix @@ -167,6 +167,8 @@ ]; hypervisorsWithUserNet = [ "qemu" "kvmtool" "vfkit" ]; hypervisorsDarwinOnly = [ "vfkit" ]; + # Hypervisors that work on darwin (qemu via HVF, vfkit natively) + hypervisorsOnDarwin = [ "qemu" "vfkit" ]; hypervisorsWithTap = builtins.filter # vfkit supports networking, but does not support tap (hv: hv != "vfkit") @@ -175,8 +177,9 @@ isDarwinOnly = hypervisor: builtins.elem hypervisor hypervisorsDarwinOnly; isDarwinSystem = system: lib.hasSuffix "-darwin" system; hypervisorSupportsSystem = hypervisor: system: - # Darwin-only hypervisors only work on darwin, others work everywhere - !(isDarwinOnly hypervisor && !(isDarwinSystem system)); + if isDarwinSystem system + then builtins.elem hypervisor hypervisorsOnDarwin + else !(isDarwinOnly hypervisor); makeExample = { system, hypervisor, config ? {} }: lib.nixosSystem {