Merge pull request #4137 from unsuman/driver/krun

Support for `libkrun` using `krunkit`

Author: AkihiroSuda

cmd/lima-driver-krunkit/main_darwin_arm64.go

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"context"
+
+	"github.com/lima-vm/lima/v2/pkg/driver/external/server"
+	"github.com/lima-vm/lima/v2/pkg/driver/krunkit"
+)
+
+// To be used as an external driver for Lima.
+func main() {
+	server.Serve(context.Background(), krunkit.New())
+}

pkg/driver/krunkit/errors_darwin_arm64.go

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package krunkit
+
+import "errors"
+
+var errUnimplemented = errors.New("unimplemented by the krunkit driver")

pkg/driver/krunkit/hack/install-vulkan-gpu.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright The Lima Authors
+# SPDX-License-Identifier: Apache-2.0
+
+set -eux -o pipefail
+
+# Install required packages
+dnf install -y dnf-plugins-core dnf-plugin-versionlock llvm18-libs
+
+# Install Vulkan and Mesa base packages
+dnf install -y \
+	mesa-vulkan-drivers \
+	vulkan-loader-devel \
+	vulkan-headers \
+	vulkan-tools \
+	vulkan-loader \
+	glslc
+
+# Enable COPR repo with patched Mesa for Venus support
+dnf copr enable -y slp/mesa-krunkit fedora-40-aarch64
+
+# Downgrade to patched Mesa version from COPR
+dnf downgrade -y mesa-vulkan-drivers.aarch64 \
+	--repo=copr:copr.fedorainfracloud.org:slp:mesa-krunkit
+
+# Lock Mesa version to prevent automatic upgrades
+dnf versionlock add mesa-vulkan-drivers
+
+# Clean up
+dnf clean all
+
+echo "Krunkit GPU(Venus) setup complete. Verify Vulkan installation by running 'vulkaninfo --summary'."

pkg/driver/krunkit/krunkit_darwin_arm64.go

@@ -0,0 +1,243 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package krunkit
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+
+	"github.com/docker/go-units"
+	"github.com/sirupsen/logrus"
+
+	"github.com/lima-vm/lima/v2/pkg/driver/vz"
+	"github.com/lima-vm/lima/v2/pkg/imgutil/proxyimgutil"
+	"github.com/lima-vm/lima/v2/pkg/iso9660util"
+	"github.com/lima-vm/lima/v2/pkg/limatype"
+	"github.com/lima-vm/lima/v2/pkg/limatype/filenames"
+	"github.com/lima-vm/lima/v2/pkg/limayaml"
+	"github.com/lima-vm/lima/v2/pkg/networks"
+	"github.com/lima-vm/lima/v2/pkg/networks/usernet"
+	"github.com/lima-vm/lima/v2/pkg/store"
+)
+
+const logLevelInfo = "3"
+
+// Cmdline constructs the command line arguments for krunkit based on the instance configuration.
+func Cmdline(inst *limatype.Instance) (*exec.Cmd, error) {
+	memBytes, err := units.RAMInBytes(*inst.Config.Memory)
+	if err != nil {
+		return nil, err
+	}
+
+	args := []string{
+		// Memory in MiB
+		"--memory", strconv.FormatInt(memBytes/units.MiB, 10),
+		"--cpus", fmt.Sprintf("%d", *inst.Config.CPUs),
+		"--device", fmt.Sprintf("virtio-serial,logFilePath=%s", filepath.Join(inst.Dir, filenames.SerialLog)),
+		"--krun-log-level", logLevelInfo,
+		"--restful-uri", "none://",
+
+		// First virtio-blk device is the boot disk
+		"--device", fmt.Sprintf("virtio-blk,path=%s,format=raw", filepath.Join(inst.Dir, filenames.DiffDisk)),
+		"--device", fmt.Sprintf("virtio-blk,path=%s", filepath.Join(inst.Dir, filenames.CIDataISO)),
+	}
+
+	// Add additional disks
+	if len(inst.Config.AdditionalDisks) > 0 {
+		ctx := context.Background()
+		diskUtil := proxyimgutil.NewDiskUtil(ctx)
+		for _, d := range inst.Config.AdditionalDisks {
+			disk, derr := store.InspectDisk(d.Name)
+			if derr != nil {
+				return nil, fmt.Errorf("failed to load disk %q: %w", d.Name, derr)
+			}
+			if disk.Instance != "" {
+				return nil, fmt.Errorf("failed to run attach disk %q, in use by instance %q", disk.Name, disk.Instance)
+			}
+			if lerr := disk.Lock(inst.Dir); lerr != nil {
+				return nil, fmt.Errorf("failed to lock disk %q: %w", d.Name, lerr)
+			}
+			extraDiskPath := filepath.Join(disk.Dir, filenames.DataDisk)
+			logrus.Infof("Mounting disk %q on %q", disk.Name, disk.MountPoint)
+			if cerr := diskUtil.ConvertToRaw(ctx, extraDiskPath, extraDiskPath, nil, true); cerr != nil {
+				return nil, fmt.Errorf("failed to convert extra disk %q to raw: %w", extraDiskPath, cerr)
+			}
+			args = append(args, "--device", fmt.Sprintf("virtio-blk,path=%s,format=raw", extraDiskPath))
+		}
+	}
+
+	// Network commands
+	networkArgs, err := buildNetworkArgs(inst)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build network arguments: %w", err)
+	}
+
+	// File sharing commands
+	if *inst.Config.MountType == limatype.VIRTIOFS {
+		for i, mount := range inst.Config.Mounts {
+			if _, err := os.Stat(mount.Location); errors.Is(err, os.ErrNotExist) {
+				if err := os.MkdirAll(mount.Location, 0o750); err != nil {
+					return nil, err
+				}
+			}
+			tag := fmt.Sprintf("mount%d", i)
+			mountArg := fmt.Sprintf("virtio-fs,sharedDir=%s,mountTag=%s", mount.Location, tag)
+			args = append(args, "--device", mountArg)
+		}
+	}
+
+	args = append(args, networkArgs...)
+	cmd := exec.CommandContext(context.Background(), vmType, args...)
+
+	return cmd, nil
+}
+
+func buildNetworkArgs(inst *limatype.Instance) ([]string, error) {
+	var args []string
+
+	// Configure default usernetwork with limayaml.MACAddress(inst.Dir) for eth0 interface
+	firstUsernetIndex := limayaml.FirstUsernetIndex(inst.Config)
+	if firstUsernetIndex == -1 {
+		// slirp network using gvisor netstack
+		krunkitSock, err := usernet.SockWithDirectory(inst.Dir, "", usernet.FDSock)
+		if err != nil {
+			return nil, err
+		}
+		client, err := vz.PassFDToUnix(krunkitSock)
+		if err != nil {
+			return nil, err
+		}
+
+		args = append(args, "--device", fmt.Sprintf("virtio-net,type=unixgram,fd=%d,mac=%s", client.Fd(), limayaml.MACAddress(inst.Dir)))
+	}
+
+	for _, nw := range inst.Networks {
+		var sock string
+		var mac string
+		if nw.Lima != "" {
+			nwCfg, err := networks.LoadConfig()
+			if err != nil {
+				return nil, err
+			}
+			switch nw.Lima {
+			case networks.ModeUserV2:
+				sock, err = usernet.Sock(nw.Lima, usernet.QEMUSock)
+				if err != nil {
+					return nil, err
+				}
+				mac = limayaml.MACAddress(inst.Dir)
+			case networks.ModeShared, networks.ModeBridged:
+				socketVMNetInstalled, err := nwCfg.IsDaemonInstalled(networks.SocketVMNet)
+				if err != nil {
+					return nil, err
+				}
+				if !socketVMNetInstalled {
+					return nil, errors.New("socket_vmnet is not installed")
+				}
+				sock, err = networks.Sock(nw.Lima)
+				if err != nil {
+					return nil, err
+				}
+				mac = nw.MACAddress
+			default:
+				return nil, fmt.Errorf("invalid network spec %+v", nw)
+			}
+		} else if nw.Socket != "" {
+			sock = nw.Socket
+			mac = nw.MACAddress
+		} else {
+			return nil, fmt.Errorf("invalid network spec %+v", nw)
+		}
+
+		device := fmt.Sprintf("virtio-net,type=unixstream,path=%s,mac=%s", sock, mac)
+		args = append(args, "--device", device)
+	}
+
+	if len(args) == 0 {
+		return args, errors.New("no socket_vmnet networks defined")
+	}
+
+	return args, nil
+}
+
+func EnsureDisk(ctx context.Context, inst *limatype.Instance) error {
+	diffDisk := filepath.Join(inst.Dir, filenames.DiffDisk)
+	if _, err := os.Stat(diffDisk); err == nil || !errors.Is(err, os.ErrNotExist) {
+		// disk is already ensured
+		return err
+	}
+
+	diskUtil := proxyimgutil.NewDiskUtil(ctx)
+
+	baseDisk := filepath.Join(inst.Dir, filenames.BaseDisk)
+
+	diskSize, _ := units.RAMInBytes(*inst.Config.Disk)
+	if diskSize == 0 {
+		return nil
+	}
+	isBaseDiskISO, err := iso9660util.IsISO9660(baseDisk)
+	if err != nil {
+		return err
+	}
+	if isBaseDiskISO {
+		// Create an empty data volume (sparse)
+		diffDiskF, err := os.Create(diffDisk)
+		if err != nil {
+			return err
+		}
+
+		err = diskUtil.MakeSparse(ctx, diffDiskF, 0)
+		if err != nil {
+			diffDiskF.Close()
+			return fmt.Errorf("failed to create sparse diff disk %q: %w", diffDisk, err)
+		}
+		return diffDiskF.Close()
+	}
+
+	// Krunkit also supports qcow2 disks but raw is faster to create and use.
+	if err = diskUtil.ConvertToRaw(ctx, baseDisk, diffDisk, &diskSize, false); err != nil {
+		return fmt.Errorf("failed to convert %q to a raw disk %q: %w", baseDisk, diffDisk, err)
+	}
+	return err
+}
+
+func startUsernet(ctx context.Context, inst *limatype.Instance) (*usernet.Client, context.CancelFunc, error) {
+	if firstUsernetIndex := limayaml.FirstUsernetIndex(inst.Config); firstUsernetIndex != -1 {
+		return usernet.NewClientByName(inst.Config.Networks[firstUsernetIndex].Lima), nil, nil
+	}
+	// Start a in-process gvisor-tap-vsock
+	endpointSock, err := usernet.SockWithDirectory(inst.Dir, "", usernet.EndpointSock)
+	if err != nil {
+		return nil, nil, err
+	}
+	krunkitSock, err := usernet.SockWithDirectory(inst.Dir, "", usernet.FDSock)
+	if err != nil {
+		return nil, nil, err
+	}
+	os.RemoveAll(endpointSock)
+	os.RemoveAll(krunkitSock)
+	ctx, cancel := context.WithCancel(ctx)
+	err = usernet.StartGVisorNetstack(ctx, &usernet.GVisorNetstackOpts{
+		MTU:      1500,
+		Endpoint: endpointSock,
+		FdSocket: krunkitSock,
+		Async:    true,
+		DefaultLeases: map[string]string{
+			networks.SlirpIPAddress: limayaml.MACAddress(inst.Dir),
+		},
+		Subnet: networks.SlirpNetwork,
+	})
+	if err != nil {
+		defer cancel()
+		return nil, nil, err
+	}
+	subnetIP, _, err := net.ParseCIDR(networks.SlirpNetwork)
+	return usernet.NewClient(endpointSock, subnetIP), cancel, err
+}