Skip to content

Commit 46a20d1

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request #3858 from norio-nomura/use-rosetta-aot-caching-with-cdi
feature: use Rosetta AOT Caching with CDI
2 parents e0a1b22 + 9d5b012 commit 46a20d1

File tree

6 files changed

+172
-3
lines changed

6 files changed

+172
-3
lines changed

‎pkg/cidata/cidata.TEMPLATE.d/boot/40-install-containerd.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ]; then
4545
mkdir -p /etc/containerd /etc/buildkit
4646
cat >"/etc/containerd/config.toml" <<EOF
4747
version = 2
48+
[plugins."io.containerd.grpc.v1.cri"]
49+
enable_cdi = true
4850
[proxy_plugins]
4951
[proxy_plugins."stargz"]
5052
type = "snapshot"
@@ -67,6 +69,8 @@ if [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
6769
mkdir -p "${LIMA_CIDATA_HOME}/.config/containerd"
6870
cat >"${LIMA_CIDATA_HOME}/.config/containerd/config.toml" <<EOF
6971
version = 2
72+
[plugins."io.containerd.grpc.v1.cri"]
73+
enable_cdi = true
7074
[proxy_plugins]
7175
[proxy_plugins."fuse-overlayfs"]
7276
type = "snapshot"

‎pkg/driver/vz/boot/05-rosetta-volume.sh‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,81 @@ else
3131
# remove binfmt.d(5) configuration if it exists
3232
[ ! -f "$binfmtd_conf" ] || rm "$binfmtd_conf"
3333
fi
34+
35+
if [ -x /mnt/lima-rosetta/rosettad ]; then
36+
CACHE_DIRECTORY=/var/cache/rosettad
37+
DEFAULT_SOCKET=${CACHE_DIRECTORY}/uds/rosetta.sock
38+
EXPECTED_SOCKET=/run/rosettad/rosetta.sock
39+
40+
# Create rosettad service
41+
if [ -f /sbin/openrc-run ]; then
42+
cat >/etc/init.d/rosettad <<EOF
43+
#!/sbin/openrc-run
44+
name="rosettad"
45+
description="Rosetta AOT Caching Daemon"
46+
required_dirs=/mnt/lima-rosetta
47+
required_files=/mnt/lima-rosetta/rosettad
48+
command=/mnt/lima-rosetta/rosettad
49+
command_args="daemon ${CACHE_DIRECTORY}"
50+
command_background=true
51+
pidfile="/run/rosettad.pid"
52+
start_pre() {
53+
# To detect creation of the socket by rosettad, remove the old socket before starting
54+
test ! -e "${DEFAULT_SOCKET}" || rm -f "${DEFAULT_SOCKET}"
55+
}
56+
start_post() {
57+
# Set the socket permission to world-writable
58+
while ! chmod -f go+w "${DEFAULT_SOCKET}"; do sleep 1; done
59+
# Create the symlink as expected by the configuration to enable Rosetta AOT caching
60+
mkdir -p "$(dirname "${EXPECTED_SOCKET}")"
61+
ln -sf "${DEFAULT_SOCKET}" "${EXPECTED_SOCKET}"
62+
}
63+
EOF
64+
chmod 755 /etc/init.d/rosettad
65+
rc-update add rosettad default
66+
rc-service rosettad start
67+
else
68+
cat >/etc/systemd/system/rosettad.service <<EOF
69+
[Unit]
70+
Description=Rosetta AOT Caching Daemon
71+
RequiresMountsFor=/mnt/lima-rosetta
72+
[Service]
73+
RuntimeDirectory=rosettad
74+
CacheDirectory=rosettad
75+
# To detect creation of the socket by rosettad, remove the old socket
76+
ExecStartPre=sh -c "test ! -e \"${DEFAULT_SOCKET}\" || rm -f \"${DEFAULT_SOCKET}\""
77+
ExecStart=/mnt/lima-rosetta/rosettad daemon "${CACHE_DIRECTORY}"
78+
# Set the socket permission to world-writable and create the symlink as expected by the configuration to enable Rosetta AOT caching.
79+
ExecStartPost=sh -c "while ! chmod -f go+w \"${DEFAULT_SOCKET}\"; do sleep 1; done; ln -sf \"${DEFAULT_SOCKET}\" \"${EXPECTED_SOCKET}\""
80+
OOMPolicy=continue
81+
OOMScoreAdjust=-500
82+
[Install]
83+
WantedBy=default.target
84+
EOF
85+
systemctl is-enabled rosettad || systemctl enable --now rosettad
86+
fi
87+
88+
# Create CDI configuration for Rosetta
89+
mkdir -p /etc/cdi /var/run/cdi /etc/buildkit/cdi
90+
cat >/etc/cdi/rosetta.yaml <<EOF
91+
cdiVersion: "0.6.0"
92+
kind: "lima-vm.io/rosetta"
93+
devices:
94+
- name: cached
95+
containerEdits:
96+
mounts:
97+
- hostPath: /var/cache/rosettad/uds/rosetta.sock
98+
containerPath: /run/rosettad/rosetta.sock
99+
options: [bind]
100+
annotations:
101+
org.mobyproject.buildkit.device.autoallow: true
102+
EOF
103+
# nerdctl requires user-specific CDI configuration directories
104+
mkdir -p "${LIMA_CIDATA_HOME}/.config/cdi"
105+
ln -sf /etc/cdi/rosetta.yaml "${LIMA_CIDATA_HOME}/.config/cdi/"
106+
chown -R "${LIMA_CIDATA_USER}" "${LIMA_CIDATA_HOME}/.config"
107+
else
108+
# Remove CDI configuration for Rosetta AOT Caching
109+
[ ! -f /etc/cdi/rosetta.yaml ] || rm /etc/cdi/rosetta.yaml
110+
[ ! -d "${LIMA_CIDATA_HOME}/.config/cdi/rosetta.yaml" ] || rm "${LIMA_CIDATA_HOME}/.config/cdi/rosetta.yaml"
111+
fi

‎pkg/driver/vz/rosetta_directory_share_arm64.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func createRosettaDirectoryShareConfiguration() (*vz.VirtioFileSystemDeviceConfi
4444
return nil, fmt.Errorf("failed to get macOS product version: %w", err)
4545
}
4646
if !macOSProductVersion.LessThan(*semver.New("14.0.0")) {
47-
cachingOption, err := vz.NewLinuxRosettaAbstractSocketCachingOptions("rosetta")
47+
cachingOption, err := vz.NewLinuxRosettaUnixSocketCachingOptions("/run/rosettad/rosetta.sock")
4848
if err != nil {
4949
return nil, fmt.Errorf("failed to create a new rosetta directory share caching option: %w", err)
5050
}

‎templates/docker-rootful.yaml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ provision:
3838
expression: .Socket.SocketUser="{{.User}}"
3939
- mode: yq
4040
path: "/etc/docker/daemon.json"
41-
expression: .features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
41+
expression: |
42+
.features.cdi = true |
43+
.features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
4244
probes:
4345
- script: |
4446
#!/bin/bash
@@ -67,5 +69,22 @@ message: |
6769
docker context use lima-{{.Name}}
6870
docker run hello-world
6971
------
72+
{{- if .Instance.Config.VMOpts.VZ.Rosetta.Enabled}}
73+
Rosetta is enabled in this VM, so you can run x86_64 containers on Apple Silicon.
74+
You can use Rosetta AOT Caching with the CDI spec:
75+
- To run a container, add `--device=lima-vm.io/rosetta=cached` to your `docker run` command:
76+
------
77+
docker run --platform=linux/amd64 --device=lima-vm.io/rosetta=cached ...
78+
------
79+
- To build an image, add `# syntax=docker/dockerfile:1-labs` at the top of your Dockerfile,
80+
and use `--device=lima-vm.io/rosetta=cached` in the `RUN` command:
81+
------
82+
# syntax=docker/dockerfile:1-labs
83+
FROM ...
84+
...
85+
RUN --device=lima-vm.io/rosetta=cached <your amd64 command>
86+
------
87+
See: https://lima-vm.io/docs/config/multi-arch/#rosetta-aot-caching
88+
{{- end}}
7089
param:
7190
containerdSnapshotter: true

‎templates/docker.yaml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ provision:
3737
apt-get install -y uidmap dbus-user-session
3838
- mode: yq
3939
path: "{{.Home}}/.config/docker/daemon.json"
40-
expression: .features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
40+
expression: |
41+
.features.cdi = true |
42+
.features.containerd-snapshotter = {{.Param.containerdSnapshotter}}
4143
owner: "{{.User}}"
4244
- mode: user
4345
script: |
@@ -74,5 +76,22 @@ message: |
7476
docker context use lima-{{.Name}}
7577
docker run hello-world
7678
------
79+
{{- if .Instance.Config.VMOpts.VZ.Rosetta.Enabled}}
80+
Rosetta is enabled in this VM, so you can run x86_64 containers on Apple Silicon.
81+
You can use Rosetta AOT Caching with the CDI spec:
82+
- To run a container, add `--device=lima-vm.io/rosetta=cached` to your `docker run` command:
83+
------
84+
docker run --platform=linux/amd64 --device=lima-vm.io/rosetta=cached ...
85+
------
86+
- To build an image, add `# syntax=docker/dockerfile:1-labs` at the top of your Dockerfile,
87+
and use `--device=lima-vm.io/rosetta=cached` in the `RUN` command:
88+
------
89+
# syntax=docker/dockerfile:1-labs
90+
FROM ...
91+
...
92+
RUN --device=lima-vm.io/rosetta=cached <your amd64 command>
93+
------
94+
See: https://lima-vm.io/docs/config/multi-arch/#rosetta-aot-caching
95+
{{- end}}
7796
param:
7897
containerdSnapshotter: true

‎website/content/en/docs/config/multi-arch.md‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,3 +86,52 @@ rosetta:
8686
```
8787
{{% /tab %}}
8888
{{< /tabpane >}}
89+
90+
### [Enable Rosetta AOT Caching with CDI spec](#rosetta-aot-caching)
91+
| ⚡ Requirement | Lima >= 2.0, macOS >= 14.0, ARM |
92+
|-------------------|----------------------------------|
93+
94+
Rosetta AOT Caching speeds up containers by saving translated binaries, so they don't need to be translated again.
95+
Learn more: [WWDC2023 video](https://developer.apple.com/videos/play/wwdc2023/10007/?time=721)
96+
97+
**How to use Rosetta AOT Caching:**
98+
99+
- **Run a container:**
100+
Add `--device=lima-vm.io/rosetta=cached` to your `docker run` command:
101+
```bash
102+
docker run --platform=linux/amd64 --device=lima-vm.io/rosetta=cached ...
103+
```
104+
105+
- **Build an image:**
106+
Add `# syntax=docker/dockerfile:1-labs` at the top of your Dockerfile to enable the `--device` option.
107+
Use `--device=lima-vm.io/rosetta=cached` in your `RUN` command:
108+
```Dockerfile
109+
# syntax=docker/dockerfile:1-labs
110+
FROM ...
111+
...
112+
RUN --device=lima-vm.io/rosetta=cached <your amd64 command>
113+
```
114+
115+
- **Check if caching works:**
116+
Look for cache files in the VM:
117+
```bash
118+
limactl shell {{.Name}} ls -la /var/cache/rosettad
119+
docker run --platform linux/amd64 --device=lima-vm.io/rosetta=cached ubuntu echo hello
120+
limactl shell {{.Name}} ls -la /var/cache/rosettad
121+
# You should see *.aotcache files here
122+
```
123+
124+
- **Check if Docker recognizes the CDI device:**
125+
Look for CDI info in the output of `docker info`:
126+
```console
127+
docker info
128+
...
129+
CDI spec directories:
130+
/etc/cdi
131+
/var/run/cdi
132+
Discovered Devices:
133+
cdi: lima-vm.io/rosetta=cached
134+
```
135+
136+
- **Learn more about CDI:**
137+
[CDI spec documentation](https://github.com/cncf-tags/container-device-interface/blob/main/SPEC.md)

0 commit comments

Comments
 (0)