Docker Image Sync Script

README.md

Docker Image Sync Script

Direct host-to-host Docker image transfer utility that preserves layer caching without requiring a registry. Useful for transferring locally built images to remote hosts while maintaining Docker's layer efficiency.

Features

• Preserves layer caching by transferring only missing layers
• Parallel blob transfer with SSH optimization
• No registry required

Usage

./docker-sync.sh -h remote_host -u remote_user -i image_name

docker-image-sync.sh

#!/bin/bash

set -euo pipefail

# Configuration variables and defaults
REMOTE_HOST=""
REMOTE_USER=""
IMAGE_NAME=""
LOCAL_STORE="$HOME/docker-images"
REMOTE_STORE=""
MAX_PARALLEL=4

# SSH configuration for optimized transfer
SSH_OPTS="-o Compression=no -o TCPKeepAlive=yes -o ServerAliveInterval=60 -o ControlMaster=auto -o ControlPath=/tmp/ssh-%r@%h:%p -o ControlPersist=1h"

# Command-line argument parsing
usage() {
	echo "Usage: $0 -h remote_host -u remote_user -i image_name"
	echo "  -h: Remote host to sync image to"
	echo "  -u: Remote user for SSH connection"
	echo "  -i: Docker image name to sync"
	exit 1
}

while getopts "h:u:i:" opt; do
	case $opt in
	h) REMOTE_HOST="$OPTARG" ;;
	u) REMOTE_USER="$OPTARG" ;;
	i) IMAGE_NAME="$OPTARG" ;;
	*) usage ;;
	esac
done

if [[ -z "$REMOTE_HOST" || -z "$REMOTE_USER" || -z "$IMAGE_NAME" ]]; then
	usage
fi

REMOTE_STORE=$(ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "echo \$HOME/docker-images")

# Smart image comparison between hosts to avoid unnecessary transfers
echo "Checking if image sync is needed..."

normalize_json() {
	echo "$1" | jq -cS '.'
}

LOCAL_INSPECT=$(docker inspect "${IMAGE_NAME}" 2>/dev/null || echo "[]")
if [[ "$LOCAL_INSPECT" == "[]" ]]; then
	echo "Error: Local image ${IMAGE_NAME} not found"
	exit 1
fi

LOCAL_CONFIG=$(echo "$LOCAL_INSPECT" | jq -cS '.[0].Config | del(.Image)')
LOCAL_ROOTFS=$(echo "$LOCAL_INSPECT" | jq -cS '.[0].RootFS')

if [[ -z "$LOCAL_CONFIG" || -z "$LOCAL_ROOTFS" || "$LOCAL_CONFIG" == "null" || "$LOCAL_ROOTFS" == "null" ]]; then
	echo "Error: Failed to extract config or rootfs data from local image"
	exit 1
fi

REMOTE_INSPECT=$(ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "docker inspect ${IMAGE_NAME} 2>/dev/null || echo '[]'")

if [[ "$REMOTE_INSPECT" != "[]" ]]; then
	REMOTE_CONFIG=$(echo "$REMOTE_INSPECT" | jq -cS '.[0].Config | del(.Image)')
	REMOTE_ROOTFS=$(echo "$REMOTE_INSPECT" | jq -cS '.[0].RootFS')

	if [[ -n "$REMOTE_CONFIG" && -n "$REMOTE_ROOTFS" && "$REMOTE_CONFIG" != "null" && "$REMOTE_ROOTFS" != "null" ]]; then
		LOCAL_CONFIG_NORM=$(normalize_json "$LOCAL_CONFIG")
		REMOTE_CONFIG_NORM=$(normalize_json "$REMOTE_CONFIG")
		LOCAL_ROOTFS_NORM=$(normalize_json "$LOCAL_ROOTFS")
		REMOTE_ROOTFS_NORM=$(normalize_json "$REMOTE_ROOTFS")

		if [[ "$LOCAL_CONFIG_NORM" == "$REMOTE_CONFIG_NORM" ]] && [[ "$LOCAL_ROOTFS_NORM" == "$REMOTE_ROOTFS_NORM" ]]; then
			echo "Images are identical on both hosts. Skipping sync."
			exit 0
		else
			echo "Images differ. Proceeding with sync..."
		fi
	else
		echo "Invalid remote image data. Proceeding with sync..."
	fi
else
	echo "Image not found on remote host. Proceeding with sync..."
fi

# Set up persistent SSH connection for efficient transfers
ssh ${SSH_OPTS} -nNf "${REMOTE_USER}@${REMOTE_HOST}"

# Utility function for parallel blob transfer
transfer_blobs() {
	local blob_list="$1"
	local source_dir="$2"
	local target_dir="$3"

	COPYFILE_DISABLE=1 tar --no-xattrs -czf - -C "$source_dir" $blob_list |
		ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "cd '$target_dir' && tar xzf -"
}

# Initialize local and remote storage directories
mkdir -p "${LOCAL_STORE}"
ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p ${REMOTE_STORE}"

IMAGE_DIR="${IMAGE_NAME//[:\/]/_}"
FULL_PATH="${LOCAL_STORE}/${IMAGE_DIR}"
REMOTE_FULL_PATH="${REMOTE_STORE}/${IMAGE_DIR}"

# Export and extract Docker image locally
echo "Saving Docker image to ${FULL_PATH}..."
mkdir -p "${FULL_PATH}"
docker save "${IMAGE_NAME}" -o "${FULL_PATH}/image.tar"

echo "Extracting image..."
cd "${FULL_PATH}"
tar xf image.tar
chmod -R a+rX "${FULL_PATH}"
rm image.tar

# Prepare remote storage structure
echo "Creating remote directory..."
ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p ${REMOTE_FULL_PATH}/blobs/sha256"

# Transfer image metadata
echo "Copying metadata files..."
COPYFILE_DISABLE=1 tar --no-xattrs -czf - index.json manifest.json oci-layout |
	ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "cd '${REMOTE_FULL_PATH}' && tar xzf -"

# Determine which blobs need to be transferred
echo "Creating list of local blobs..."
cd "${FULL_PATH}/blobs/sha256"
local_blobs=$(ls)

echo "Getting list of remote blobs..."
remote_blobs=$(ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "ls ${REMOTE_FULL_PATH}/blobs/sha256 2>/dev/null || echo ''")

blobs_to_transfer=()
for blob in $local_blobs; do
	if [ -f "$blob" ]; then
		if ! echo "$remote_blobs" | grep -q "^${blob}$"; then
			blobs_to_transfer+=("$blob")
		else
			echo "Blob ${blob} already exists"
		fi
	fi
done

# Execute parallel blob transfer in batches
if [ ${#blobs_to_transfer[@]} -gt 0 ]; then
	echo "Transferring ${#blobs_to_transfer[@]} blobs..."

	batch_size=$(((${#blobs_to_transfer[@]} + MAX_PARALLEL - 1) / MAX_PARALLEL))
	if [ $batch_size -lt 1 ]; then
		batch_size=1
	fi

	for ((i = 0; i < ${#blobs_to_transfer[@]}; i += batch_size)); do
		batch_end=$((i + batch_size))
		if [ $batch_end -gt ${#blobs_to_transfer[@]} ]; then
			batch_end=${#blobs_to_transfer[@]}
		fi

		batch_list="${blobs_to_transfer[@]:i:batch_size}"
		echo "Transferring batch $(((i / batch_size) + 1))..."
		transfer_blobs "$batch_list" "." "${REMOTE_FULL_PATH}/blobs/sha256" &
	done

	wait
fi

# Clean up unnecessary remote blobs
echo "Cleaning up obsolete blobs..."
for blob in $remote_blobs; do
	if ! echo "$local_blobs" | grep -q "^${blob}$"; then
		echo "Removing obsolete blob: ${blob}"
		ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_FULL_PATH}/blobs/sha256/${blob}"
	fi
done

ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "
    if [ -d ${REMOTE_FULL_PATH}/blobs/sha256 ] && [ -z \"\$(ls -A ${REMOTE_FULL_PATH}/blobs/sha256)\" ]; then
        rm -rf ${REMOTE_FULL_PATH}/blobs
    fi
"

# Load synced image on remote host and clean up
echo "Loading image on remote host..."
ssh ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_HOST}" "cd ${REMOTE_FULL_PATH} && COPYFILE_DISABLE=1 tar --no-xattrs -cf - . | docker load"

ssh -O exit -o ControlPath=/tmp/ssh-%r@%h:%p "${REMOTE_USER}@${REMOTE_HOST}" 2>/dev/null || true

echo "Image sync completed successfully!"