Sync for s3cmd put with skip existing and delay after upload each file

s3-delay-sync.sh

#!/bin/bash

# Written by Jolene Engo <[email protected]>

# This script is useful for syncing files to an S3 bucket with a delay
# Great for situations where a remote process is grabbing files after upload
# and you don't want to overwhelm the remote system
# The delay will slow the syncing
# If the file already exists, it will be skippped.  This allows a resume
# that s3cmd wouldn't normally support for put

# File glob to check for (example *.gz)
GLOB="*"
# When files are missing, this is how long to wait after upload
DELAY=15
# s3 bucket prefix (bucket and path but no trailing slash)
S3BUCKET="s3://example-s3-bucket/mypath"

if ! [ -x "$(command -v s3cmd)" ]; then
	echo "Error: Could not find s3cmd";
	exit 1
fi

for f in $GLOB; do
	echo "Checking for $f";
	e=`s3cmd ls $S3BUCKET/$f`

	if [ "$e" == "" ]; then
		echo "Uploading";
		s3cmd put $f $S3BUCKET/;
		echo "Upload complete waiting $DELAY seconds";
		sleep $DELAY;
	else
		echo "File found skipping";
	fi
done