M3 MacBook Pro - Ubuntu VM running k3s/cilium/kata-qemu

Install tart to get a ubuntu VM with nested vert.

brew install cirruslabs/cli/tart
tart clone ghcr.io/cirruslabs/ubuntu:latest ubuntu
tart run ubuntu --nested

SSH in

ssh admin@$(tart ip ubuntu)
# Password is "admin"

Run the script to install everything:

# Install and run k3s server.
# Nodes won't be ready yet due to having no CNI.
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='--flannel-backend=none --disable-network-policy' sh -

# Own kubeconfig
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
sudo chown $(id -u):$(id -g) $KUBECONFIG

# Install Cilium CLI 
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}

# Install Cilium
cilium install --version 1.18.2 --set=ipam.operator.clusterPoolIPv4PodCIDRList="10.42.0.0/16"
cilium status --wait

# Install qemu
sudo apt-get update
sudo apt-get install -y qemu-system qemu-efi-aarch64 qemu-kvm

# Install helm
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# Install kata-containers via helm
export VERSION=$(curl -sSL https://api.github.com/repos/kata-containers/kata-containers/releases/latest | jq .tag_name | tr -d '"')
export CHART="oci://ghcr.io/kata-containers/kata-deploy-charts/kata-deploy"
helm install kata-deploy "${CHART}" \
    --version "${VERSION}" \
    --namespace kube-system \
    --wait --timeout 10m --atomic \
    --set k8sDistribution=k3s \
    --set env.shims=qemu

# Wait for kata-deploy to be ready
kubectl wait --for=condition=ready pod \
    -l name=kata-deploy \
    -n kube-system \
    --timeout=120s

# Wait for kata-containers to create its config file
while [ ! -f /opt/kata/share/defaults/kata-containers/configuration-qemu.toml ]; do
  echo 'Waiting to edit kata-containers config...'
  sleep 1
done

# Disable pmu=off in kata containers - not supported on Apple Silicon
sudo sed -i 's/^cpu_features="pmu=off"/cpu_features=""/' /opt/kata/share/defaults/kata-containers/configuration-qemu.toml

# Add kata-containers to k3s containerd config
sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl > /dev/null <<'EOF'

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."kata-qemu"]
  runtime_type = "io.containerd.kata.v2"
  privileged_without_host_devices = false
  pod_annotations = ["io.katacontainers.*"]
EOF

# Restart k3s to load new containerd config
sudo systemctl restart k3s

# Launch a test pod
kubectl run kata-shell --image=alpine --restart=Never --overrides='{"spec":{"runtimeClassName":"kata-qemu"}}' --command -- sh -c "sleep infinity" && \                              
kubectl wait --for=condition=Ready pod/kata-shell --timeout=120s && \
kubectl exec -it kata-shell -- uname -a && \
kubectl delete pod kata-shell

srcreigh/m3-arm-k3s-cilium-kata-qemu-guide.md