qemu-host-topo.sh

#!/bin/bash
set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
KERNEL="${SCRIPT_DIR}/linux/arch/x86/boot/bzImage"
ROOTFS="${SCRIPT_DIR}/debian-root"
QMP_SOCK="${SCRIPT_DIR}/.qemu-qmp.sock"

# Host topology: AMD EPYC 7V73X
# 1 socket, 8 CCDs (dies), 8 cores/CCD, 2 threads/core = 128 vCPUs
# L3 96MB shared per CCD (8 cores = 16 threads)
VCPUS=128
SOCKETS=1
DIES=8           # CCDs — cores in the same die share L3
CORES_PER_DIE=8
THREADS=2

if [ ! -f "$KERNEL" ]; then
    echo "Error: Kernel image not found at $KERNEL"
    exit 1
fi

if [ ! -d "$ROOTFS" ] || [ -z "$(ls -A "$ROOTFS")" ]; then
    echo "Warning: $ROOTFS is empty. Populate it with a Debian rootfs first."
    echo "  e.g.: sudo debootstrap stable $ROOTFS http://deb.debian.org/debian"
    exit 1
fi

# Clean up stale QMP socket
rm -f "$QMP_SOCK"

# ── Fork a background pinner, then exec QEMU in foreground (keeps stdin) ─
# After exec, $$ (= QEMU_PID) becomes the QEMU process.
QEMU_PID=$$

(
    # Wait for QEMU to create the QMP socket and vCPU threads.
    for _ in $(seq 1 50); do
        [ -S "$QMP_SOCK" ] && break
        sleep 0.1
    done

    if ! kill -0 $QEMU_PID 2>/dev/null; then
        echo "Error: QEMU process not found (PID=$QEMU_PID)" >&2
        exit 1
    fi

    if [ ! -S "$QMP_SOCK" ]; then
        echo "Error: QMP socket not found at $QMP_SOCK" >&2
        exit 1
    fi

    echo "Pinning $VCPUS vCPU threads to host CPUs..." >&2

    # Query QEMU directly for cpu-index -> thread-id -> topology mapping.
    # Host layout (EPYC): thread-0 on CPUs 0-63, thread-1 on CPUs 64-127.
    #   host_cpu = (die*8 + core) + thread*64
    pinned=0
    while read -r tid_num host_cpu; do
        [ -n "$tid_num" ] || continue
        taskset -pc "$host_cpu" "$tid_num" > /dev/null
        pinned=$((pinned + 1))
    done < <(
        python3 - "$QMP_SOCK" "$CORES_PER_DIE" <<'PY'
import json
import socket
import sys

qmp_sock = sys.argv[1]
cores_per_die = int(sys.argv[2])
threads_stride = 64

sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(qmp_sock)
file_obj = sock.makefile("rwb", buffering=0)

# Greeting
file_obj.readline()

for command in (
    {"execute": "qmp_capabilities"},
    {"execute": "query-cpus-fast"},
):
    file_obj.write((json.dumps(command) + "\n").encode())
    file_obj.flush()
    reply = json.loads(file_obj.readline().decode())
    if command["execute"] == "query-cpus-fast":
        for cpu in reply["return"]:
            props = cpu["props"]
            die = int(props["die-id"])
            core = int(props["core-id"])
            thread = int(props["thread-id"])
            host_cpu = die * cores_per_die + core + thread * threads_stride
            print(f"{cpu['thread-id']} {host_cpu}")

file_obj.close()
sock.close()
PY
    )

    echo "Pinned $pinned vCPU threads. QEMU PID=$QEMU_PID" >&2
) &

# ── Launch QEMU in foreground (exec replaces this process, keeping stdin) ─
exec qemu-system-x86_64 \
    -enable-kvm \
    -cpu host,host-cache-info=on,topoext=on,legacy-multi-node=on,l3-cache=on \
    -smp "${VCPUS},sockets=${SOCKETS},dies=${DIES},cores=${CORES_PER_DIE},threads=${THREADS}" \
    -m 4G \
    -kernel "$KERNEL" \
    -append "root=/dev/root rootfstype=9p rootflags=trans=virtio,version=9p2000.L rw console=ttyS0 nokaslr" \
    -fsdev local,id=rootdev,path="$ROOTFS",security_model=passthrough \
    -device virtio-9p-pci,fsdev=rootdev,mount_tag=/dev/root \
    -netdev bridge,id=net0,br=br0 \
    -device virtio-net-pci,netdev=net0 \
    -nographic \
    -qmp unix:"$QMP_SOCK",server,nowait \
    "$@"