Script to bootstrap k3s with custom containerd

k3s-bootstrap.sh

#!/bin/bash
set -euo pipefail

###
# This script is used to bootstrap the K3S server on a system with ZFS.
# It checks for the existence of ZFS datasets, creates necessary directories,
# installs containerd and nerdctl, configures containerd, and installs k3s.
# It also handles cleanup of temporary files and directories.
# It is recommended to run this script in a controlled environment.

CTRD_ZFS_DATASET="ubuntu/var/lib/containerd"
CTRD_DATA_DIR="/var/lib/containerd"
K3S_ZFS_DATASET="gluon/pv/db"
K3S_DATA_DIR="/data/pv/db/k3s"
K3S_CLUSTER_DOMAIN="bytefuse.local"
K3S_CLUSTER_CIDR="100.64.0.0/21"
K3S_SERVICE_CIDR="100.64.8.0/21"
K3S_SERVER_IP="127.0.0.1"
CILIUM_CLUSTER_IPv4_MASK_SIZE=23
CILIUM_NAMESPACE="kube-system"
INSTALL_NERDCTL=${INSTALL_NERDCTL:-false}
NERDCTL_RELEASE_URL="https://api.github.com/repos/containerd/nerdctl/releases/latest"

###
# Logging functions
SCRIPT_NAME="k3s.bootstrap"
HOSTNAME=$(hostname -s)
log() {
  local level="$1"
  local message="$2"
  local colour=""
  case "$level" in
    DEBUG) colour='\e[90m' ;; # Grey
    INFO ) colour='\e[0m' ;; # Reset
    WARN ) colour='\e[33m' ;; # Yellow
    ERROR) colour='\e[31m' ;; # Red
  esac
  printf '\e[32m%(%Y-%m-%d)T %(%T)T \e[35m%s \e[34m%s \e[90m%5s %b%s\e[0m\n' \
    -1 -1 "$HOSTNAME" "$SCRIPT_NAME[$$]" "$level" "$colour" "$message"
}

debug() { log 'DEBUG' "$1"; }
info()  { log 'INFO'  "$1"; }
warn()  { log 'WARN'  "$1"; }
error() { log 'ERROR' "$1"; }

printf "\n=====================================\n"
printf "  \e[33mK3S\e[0m Bootstrap Script\n"
printf "=====================================\n\n"

info "ZFS: 🛠️ Checking datasets and data directories"
if zfs list -o name -H "$CTRD_ZFS_DATASET" >/dev/null 2>&1; then
  info "ZFS: ✅ Dataset for containerd path exists: $CTRD_ZFS_DATASET"
else
  error "ZFS: ❌ Dataset for containerd path does not exist: $CTRD_ZFS_DATASET"
  exit 1
fi

if [ ! -d "$CTRD_DATA_DIR" ]; then
  error "ZFS: ❌ containerd directory missing: $CTRD_DATA_DIR"
  exit 1
fi

if zfs list -o name -H "$K3S_ZFS_DATASET" >/dev/null 2>&1; then
  info "ZFS: ✅ Dataset for K3S path exists: $K3S_ZFS_DATASET"
else
  error "ZFS: ❌ Dataset for K3S path does not exist: $K3S_ZFS_DATASET"
  exit 1
fi

if [ ! -d "$K3S_DATA_DIR" ]; then
  info "ZFS: 🛠️ Creating K3S data directory: $K3S_DATA_DIR"
  if sudo mkdir -p "$K3S_DATA_DIR"; then
    info "ZFS: ✅ Created K3S data directory"
  else
    error "ZFS: ❌ Failed to create K3S data directory"
    exit 1
  fi
fi

info "APT: 🛠️ Installing containerd & tools"
sudo apt update && sudo apt install -y containerd curl dpkg jq sed

if [ -f /etc/apt/sources.list.d/helm-stable.sources ] && command -v helm &> /dev/null; then
  info "APT: ✅ Helm is already installed"
else
  info "APT: 🛠️ Helm is not installed, adding Helm repository"
  printf '%s\n' \
  "Types: deb" \
  "URIs: https://baltocdn.com/helm/stable/debian/" \
  "Suites: all" \
  "Components: main" \
  "Architectures: $(dpkg --print-architecture)" \
  "Signed-By:" \
  "$(curl -s https://baltocdn.com/helm/signing.asc | sed 's/^/ /')" \
  | sudo tee /etc/apt/sources.list.d/helm-stable.sources > /dev/null
fi

if ! command -v helm &> /dev/null; then
  info "APT: 🛠️ Installing Helm"
  if sudo apt update && sudo apt install -y helm; then
    info "APT: ✅ Helm installed successfully"
  else
    error "APT: ❌ Failed to install Helm"
    exit 1
  fi
else
  info "APT: ✅ Helm is already installed"
fi

# Create a temporary directory
TEMP_DIR=$(mktemp -d)
info "TMP: 🛠️ Creating temporary directory: $TEMP_DIR"

# Check if the temporary directory was created successfully
if [ -z "$TEMP_DIR" ]; then
  error "TMP: ❌ Failed to create temporary directory: $TEMP_DIR"
  exit 1
fi

# Define a function to remove the temporary directory
cleanup() {
  info "🛠️ Cleaning up temporary directory: $TEMP_DIR"
  rm -rf "$TEMP_DIR"
  info "✅ Bootstrap script completed successfully 🎉"
}

# Trap the EXIT signal to call the cleanup function
trap cleanup EXIT

info "CTRD: 🛠️ Configuring containerd"
# Create the containerd config directory if it doesn't exist
if [ ! -d /etc/containerd ]; then
  info "CTRD: 🛠️ Creating containerd config directory"
  # Check if successed
  if sudo mkdir -p /etc/containerd; then
    info "CTRD: ✅ Created containerd config directory"
  else
    error "CTRD: ❌ Failed to create containerd config directory"
    exit 1
  fi
else 
  info "CTRD: ✅ containerd config directory already exists"
fi
# Generate the default containerd config file
# Check if the config file already exists
if [ -f /etc/containerd/config.toml ]; then
  info "CTRD: ✅ containerd config file already exists, skipping generation"
else
  info "CTRD: 🛠️ Generating containerd config file"
  containerd config default | sudo tee /etc/containerd/config.toml > /dev/null
  # You might want to ensure SystemdCgroup is true for K3s compatibility:
  sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
fi

# Check if the containerd service is enabled and running
if systemctl is-enabled containerd >/dev/null 2>&1; then
  info "CTRD: ✅ containerd service is already enabled"
else
  info "CTRD: 🛠️ Enabling containerd service"
  sudo systemctl enable --now containerd
fi
if systemctl is-active containerd >/dev/null 2>&1; then
  info "CTRD: ✅ containerd service is already running"
else
  info "CTRD: ▶️ Starting containerd service"
  sudo systemctl start containerd
fi

if $INSTALL_NERDCTL; then
  # Download the latest version of nerdctl
  info "NCTL: 🛠️ Installing latest nerdctl"

  # Ensure the download URL is correctly filtered, and handle potential errors
  NERDCTL_DOWNLOAD_URL=$(curl -s $NERDCTL_RELEASE_URL | jq -r '.assets[] | select(.name | contains("full") and contains("linux-amd64")) | .browser_download_url' | head -n 1)
  if [ -z "$NERDCTL_DOWNLOAD_URL" ]; then
    error "NCTL: ❌ Failed to find nerdctl download URL"
    exit 1
  fi
  nerd_pkg_name=$(basename "$NERDCTL_DOWNLOAD_URL")
  if curl -sSL "$NERDCTL_DOWNLOAD_URL" -o "$TEMP_DIR/$nerd_pkg_name" ; then
    info "NCTL: ✅ Downloaded nerdctl package: $nerd_pkg_name"
  else
    error "NCTL: ❌ Failed to download nerdctl package"
    exit 1
  fi
  nerd_version=$(printf $nerd_pkg_name | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?')
  info "NCTL: 🛠️ Building nerdctl package version: $nerd_version"
  mkdir -p $TEMP_DIR/nerdctl-$nerd_version-linux-amd64/DEBIAN
  mkdir -p $TEMP_DIR/nerdctl-$nerd_version-linux-amd64/usr/local
  tar -C $TEMP_DIR/nerdctl-$nerd_version-linux-amd64/usr/local -xzf "$TEMP_DIR/$nerd_pkg_name"
  cat > $TEMP_DIR/nerdctl-$nerd_version-linux-amd64/DEBIAN/control <<EOF
Package: nerdctl
Version: ${nerd_version}
Section: utils
Priority: optional
Architecture: amd64
Maintainer: [email protected]
Description: Docker-compatible CLI for containerd
EOF
  cd $TEMP_DIR
  if dpkg-deb --build nerdctl-$nerd_version-linux-amd64; then
    info "NCTL: ✅ Built nerdctl package"
    dpkg -I nerdctl-$nerd_version-linux-amd64.deb
    sudo cp nerdctl-$nerd_version-linux-amd64.deb /usr/local/src/
  else
    error "NCTL: ❌ Failed to build nerdctl package"
    exit 1
  fi
  if sudo dpkg -i nerdctl-$nerd_version-linux-amd64.deb; then
    info "NCTL: ✅ Installed nerdctl"
  else
    error "NCTL: ❌ Failed to install nerdctl"
    exit 1
  fi
else
  info "NCTL: ↪️ Skipping nerdctl installation as per configuration \$INSTALL_NERDCTL: ${INSTALL_NERDCTL}"
fi


# Install k3s stable
# Check if k3s is already installed
info "K3S: 🛠️ Checking if k3s is already installed"
if command -v k3s &> /dev/null; then
  info "K3S: ✅ k3s is already installed"
else
  info "K3S: 🛠️ Installing k3s"
  # Settings reference: https://docs.k3s.io/cli/server
  # Use external containerd socket
  # Use the ZFS dataset for data storage
  # Disable unnecessary components for a minimal setup
  curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server \
    --container-runtime-endpoint unix:///run/containerd/containerd.sock \
    --node-name ${HOSTNAME} \
    --cluster-domain ${K3S_CLUSTER_DOMAIN} \
    --flannel-backend none \
    --disable-kube-proxy \
    --disable-helm-controller \
    --disable-network-policy \
    --disable local-storage \
    --disable servicelb \
    --disable traefik \
    --cluster-cidr $K3S_CLUSTER_CIDR \
    --service-cidr $K3S_SERVICE_CIDR \
    --data-dir $K3S_DATA_DIR" \
  sh -
fi

# Wire up k3s kubeconfig for root user
if [ -d /root/.kube ] && [ -f /root/.kube/config ]; then
  info "K3S: ✅ root kubeconfig already exists"
else
  info "K3S: 🛠️ Creating .kube directory for root user"
  if sudo mkdir -p /root/.kube && sudo ln -sf /etc/rancher/k3s/k3s.yaml /root/.kube/config; then
    info "K3S: ✅ Created .kube directory and symlinked k3s config"
  else
    error "K3S: ❌ Failed to create .kube directory or symlink k3s config"
    exit 1
  fi
fi

# Install Cilium as the CNI plugin
info "HELM: 🛠️ Installing Cilium CNI plugin"

if sudo helm repo list -o json | jq -e '.[] | select(.name == "cilium")' > /dev/null; then
  info "HELM: ✅ Cilium Helm repo is already present"
else
  info "HELM: 🛠️ Adding Cilium Helm repo"
  if sudo helm repo add cilium https://helm.cilium.io ; then
    info "HELM: ✅ Added Cilium Helm repo successfully"
  else
    error "HELM: ❌ Failed to add Cilium Helm repo"
    exit 1
  fi
fi

if sudo helm repo update; then
  info "HELM: ✅ Updated Cilium Helm repo"
else
  error "HELM: ❌ Failed to update Cilium Helm repo"
  exit 1
fi

# Check if already installed
if sudo helm status cilium -n ${CILIUM_NAMESPACE} &>/dev/null; then
  info "HELM: ↪️ Cilium already installed in kube-system, skipping install."
else
  info "HELM: 🛠️ Installing Cilium in kube-system namespace"
  if sudo helm install cilium cilium/cilium \
    --namespace "$CILIUM_NAMESPACE" \
    --create-namespace \
    --set kubeProxyReplacement=true \
    --set k8sServiceHost=$K3S_SERVER_IP \
    --set k8sServicePort=6443 \
    --set cni.binPath=/opt/cni/bin \
    --set cni.confPath=/etc/cni/net.d \
    --set cni.customConf=true \
    --set ipam.mode=cluster-pool \
    --set ipam.operator.clusterPoolIPv4MaskSize=23 \
    --set ipam.operator.clusterPoolIPv4PodCIDRList="$K3S_CLUSTER_CIDR" \
    --set ipv6.enabled=false \
    --set operator.replicas=1 ; then
    info "HELM: ✅ Installed Cilium successfully"
    exit 0
  else
    error "HELM: ❌ Failed to install Cilium"
    exit 1
  fi
fi

# To uninstall Cilium, you can use the following commands:
# sudo helm uninstall cilium -n kube-system
# sudo kubectl get namespace cilium-secrets -o json   | jq 'del(.spec.finalizers)'   | sudo kubectl replace --raw "/api/v1/namespaces/cilium-secrets/finalize" -f -