Containerd unpack benchmark script - Measure container image unpacking performance with CPU affinity control

bench.sh

#!/usr/bin/env bash
# unpack-bench.sh
#
# Limit cores used during containerd "unpack" and benchmark it.
#
# Requirements: containerd (ctr), taskset, /usr/bin/time
# Usage:
#   sudo ./unpack-bench.sh --image docker.io/library/ubuntu:22.04 --cpus 2 --iters 5
# Options:
#   --image  <ref>   OCI image ref (default: docker.io/library/alpine:latest)
#   --cpus   <N>     Number of CPU cores to allow (pins to cores 0..N-1)
#   --iters  <N>     Iterations (default: 3)
#   --ns     <name>  Containerd namespace (default: benchunpack)
#   --drop-caches    Drop Linux FS caches before each run (root; system-wide)
set -euo pipefail

IMAGE="docker.io/library/alpine:latest"
CPUS=1
ITERS=3
NS="benchunpack"
DROP_CACHES=0

# --- args ---
while [[ $# -gt 0 ]]; do
  case "$1" in
    --image) IMAGE="$2"; shift 2 ;;
    --cpus)  CPUS="$2"; shift 2 ;;
    --iters) ITERS="$2"; shift 2 ;;
    --ns)    NS="$2"; shift 2 ;;
    --drop-caches) DROP_CACHES=1; shift ;;
    -h|--help)
      sed -n '1,14p' "$0"; exit 0 ;;
    *) echo "Unknown arg: $1" >&2; exit 1 ;;
  esac
done

# --- checks ---
command -v ctr >/dev/null || { echo "ctr not found"; exit 1; }
command -v taskset >/dev/null || { echo "taskset not found"; exit 1; }
[[ -x /usr/bin/time ]] || { echo "/usr/bin/time not found"; exit 1; }
[[ "$CPUS" -ge 1 ]] || { echo "--cpus must be >=1"; exit 1; }

# Check for root/sudo
if [[ $EUID -ne 0 ]]; then
   echo "Error: This script must be run with sudo or as root" >&2
   echo "Usage: sudo $0 [options]" >&2
   exit 1
fi

# Build CPU set "0,1,...,N-1"
CPUSET=$(seq -s, 0 $((CPUS-1)))

echo "== Settings =="
echo "Image:       $IMAGE"
echo "Namespace:   $NS"
echo "CPU cores:   $CPUSET  (count=$CPUS)"
echo "Iterations:  $ITERS"
echo "Drop caches: $DROP_CACHES"
echo

# Create namespace (idempotent)
sudo ctr ns create "$NS" 2>/dev/null || true

echo "== Pulling image into content store (no unpack yet) =="
# One-time pull: subsequent runs will be unpack-only
sudo ctr -n "$NS" images pull "$IMAGE" >/dev/null

times=()
for i in $(seq 1 "$ITERS"); do
  echo "--- Iter $i/$ITERS ---"

  # Clean previous snapshots (safe within dedicated namespace)
  SNAP_LIST=$(sudo ctr -n "$NS" snapshots ls 2>/dev/null | tail -n +2 | awk '{print $1}' || true)
  if [[ -n "${SNAP_LIST}" ]]; then
    echo "$SNAP_LIST" | while read snap; do
      sudo ctr -n "$NS" snapshots rm "$snap" >/dev/null 2>&1 || true
    done
  fi

  if [[ "$DROP_CACHES" -eq 1 ]]; then
    # WARNING: affects the whole system
    sudo sync
    echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
  fi

  # Time unpack with CPU affinity
  MOUNT_TARGET=$(mktemp -d)
  TFILE=$(mktemp)
  CPU_LOG="cpu_usage_iter${i}.txt"
  
  # Start CPU and I/O monitoring in background
  IO_LOG="io_usage_iter${i}.txt"
  (
    echo "# CPU and I/O usage for iteration $i (timestamp, ctr_pid, ctr_cpu%, system_cpu%, ctr_read_bytes, ctr_write_bytes, sys_read_kb_total, sys_write_kb_total)" > "$CPU_LOG"
    
    # Wait for ctr process to start
    CTR_PID=""
    for retry in {1..20}; do
      CTR_PID=$(pgrep -f "ctr -n $NS image mount" | head -1)
      if [[ -n "$CTR_PID" ]]; then
        break
      fi
      sleep 0.1
    done
    
    # Initialize I/O counters
    if [[ -n "$CTR_PID" ]] && [[ -r /proc/$CTR_PID/io ]]; then
      read_bytes_prev=$(awk '/read_bytes/ {print $2}' /proc/$CTR_PID/io 2>/dev/null || echo 0)
      write_bytes_prev=$(awk '/write_bytes/ {print $2}' /proc/$CTR_PID/io 2>/dev/null || echo 0)
    else
      read_bytes_prev=0
      write_bytes_prev=0
    fi
    
    # Get initial system I/O stats
    if [[ -r /proc/diskstats ]]; then
      sys_read_sectors_prev=$(awk '{read_sectors+=$6} END {print read_sectors+0}' /proc/diskstats)
      sys_write_sectors_prev=$(awk '{write_sectors+=$10} END {print write_sectors+0}' /proc/diskstats)
    else
      sys_read_sectors_prev=0
      sys_write_sectors_prev=0
    fi
    
    time_prev=$(date +%s.%N)
    
    if [[ -n "$CTR_PID" ]]; then
      # Monitor the specific ctr process
      while kill -0 "$CTR_PID" 2>/dev/null; do
        TIMESTAMP=$(date +%s.%N)
        # Get CPU usage for the specific ctr process
        CTR_CPU=$(ps -p "$CTR_PID" -o %cpu= 2>/dev/null | awk '{print $1+0}')
        # Get overall system CPU
        SYS_CPU=$(ps aux | awk 'NR>1 {sum+=$3} END {print sum+0}')
        
        # Simple I/O monitoring - just log current values without rate calculation
        if [[ -r /proc/$CTR_PID/io ]]; then
          read_bytes_curr=$(awk '/read_bytes/ {print $2}' /proc/$CTR_PID/io 2>/dev/null || echo 0)
          write_bytes_curr=$(awk '/write_bytes/ {print $2}' /proc/$CTR_PID/io 2>/dev/null || echo 0)
        else
          read_bytes_curr=0
          write_bytes_curr=0
        fi
        
        # System I/O from /proc/diskstats - sum all devices
        if [[ -r /proc/diskstats ]]; then
          sys_read_kb=$(awk '{sectors+=$6} END {print int(sectors*512/1024)}' /proc/diskstats 2>/dev/null || echo 0)
          sys_write_kb=$(awk '{sectors+=$10} END {print int(sectors*512/1024)}' /proc/diskstats 2>/dev/null || echo 0)
        else
          sys_read_kb=0
          sys_write_kb=0
        fi
        
        echo "$TIMESTAMP $CTR_PID $CTR_CPU $SYS_CPU $read_bytes_curr $write_bytes_curr $sys_read_kb $sys_write_kb" >> "$CPU_LOG"
        sleep 0.1
      done
    else
      # Fallback: monitor without specific PID
      while true; do
        TIMESTAMP=$(date +%s.%N)
        CTR_CPU=$(ps aux | grep -E "ctr.*mount" | grep -v grep | awk '{sum+=$3} END {print sum+0}')
        SYS_CPU=$(ps aux | awk 'NR>1 {sum+=$3} END {print sum+0}')
        echo "$TIMESTAMP 0 $CTR_CPU $SYS_CPU 0 0 0 0" >> "$CPU_LOG"
        sleep 0.1
      done &
      FALLBACK_MON=$!
      # Kill fallback monitor after reasonable time
      ( sleep 10 && kill $FALLBACK_MON 2>/dev/null ) &
    fi
  ) &
  MONITOR_PID=$!
  
  set +e
  sudo /usr/bin/time -f "%e" -o "$TFILE" \
    taskset -c "$CPUSET" \
    ctr -n "$NS" image mount "$IMAGE" "$MOUNT_TARGET" >/dev/null 2>&1
  rc=$?
  set -e
  
  # Stop CPU monitoring
  kill $MONITOR_PID 2>/dev/null || true
  wait $MONITOR_PID 2>/dev/null || true
  
  # Clean up mount point
  sudo ctr -n "$NS" image unmount "$MOUNT_TARGET" >/dev/null 2>&1 || true
  rmdir "$MOUNT_TARGET" 2>/dev/null || true
  
  if [[ $rc -ne 0 ]]; then
    echo "Mount/unpack failed (rc=$rc). See stderr by running command without redirection."
    rm -f "$TFILE"
    exit $rc
  fi
  ET=$(cat "$TFILE"); rm -f "$TFILE"
  echo "elapsed_s=$ET"
  times+=("$ET")
done

# Compute average
avg=$(printf '%s\n' "${times[@]}" | awk '{s+=$1} END{if(NR>0) printf "%.3f", s/NR; else print "NaN"}')
echo
echo "== Results =="
for idx in "${!times[@]}"; do
  printf "iter_%02d: %ss\n" "$((idx+1))" "${times[$idx]}"
done
echo "average: ${avg}s"
echo

echo "== Performance Statistics =="
for i in $(seq 1 "$ITERS"); do
  CPU_LOG="cpu_usage_iter${i}.txt"
  if [[ -f "$CPU_LOG" ]]; then
    # Simple CPU stats to avoid awk syntax issues
    SAMPLE_COUNT=$(tail -n +2 "$CPU_LOG" | wc -l)
    if [[ $SAMPLE_COUNT -gt 0 ]]; then
      # Get a sample line with PID if available
      SAMPLE_LINE=$(tail -n +2 "$CPU_LOG" | grep -E '^[0-9.]+ [0-9]+' | head -1)
      if [[ -n "$SAMPLE_LINE" ]]; then
        PID=$(echo "$SAMPLE_LINE" | awk '{print $2}')
        echo "Iter $i: PID=$PID | samples=$SAMPLE_COUNT | CPU and I/O data captured"
      else
        echo "Iter $i: samples=$SAMPLE_COUNT | Fallback monitoring mode"
      fi
    else
      echo "Iter $i: No data"
    fi
  fi
done
echo
echo "CPU usage logs saved to: cpu_usage_iter*.txt"