redis failover script

failover.sh

#!/bin/bash

# Function to get pod information
get_pod_info() {
    local pod_name=$1
    local role=$(kubectl exec -it $pod_name -c redis -- redis-cli info replication | grep role | cut -d':' -f2 | tr -d '\r')
    local pod_ip=$(kubectl get pod $pod_name -o jsonpath='{.status.podIP}')
    echo "$pod_name $role $pod_ip"
}

# Function to check replication info
check_replication_info() {
    local pod_name=$1
    echo "Replication info for $pod_name:"
    kubectl exec -it $pod_name -c redis -- redis-cli info replication
    echo "------------------------"
}

# Function to monitor slave pod until it becomes master
monitor_slave_promotion() {
    local slave_pod=$1
    echo "Monitoring $slave_pod for promotion to master..."
    while true; do
        role=$(kubectl exec -it $slave_pod -c redis -- redis-cli info replication | grep role | cut -d':' -f2 | tr -d '\r')
        echo "Current role of $slave_pod: $role"
        if [ "$role" = "master" ]; then
            echo "$slave_pod has been promoted to master!"
            break
        fi
        sleep 5
    done
}

# Function to monitor new slave synchronization
monitor_slave_sync() {
    local slave_pod=$1
    local master_ip=$2
    echo "Monitoring $slave_pod for synchronization with new master ($master_ip)..."
    while true; do
        master_host=$(kubectl exec -it $slave_pod -c redis -- redis-cli info replication | grep master_host | cut -d':' -f2 | tr -d '\r')
        master_link_status=$(kubectl exec -it $slave_pod -c redis -- redis-cli info replication | grep master_link_status | cut -d':' -f2 | tr -d '\r')
        echo "Current master_host of $slave_pod: $master_host, master_link_status: $master_link_status"
        if [ "$master_host" = "$master_ip" ] && [ "$master_link_status" = "up" ]; then
            echo "$slave_pod has successfully synchronized with the new master!"
            break
        fi
        sleep 5
    done
}

# Function to write to master and read from slave
write_and_read() {
    local master_pod=$1
    local slave_pod=$2
    local key="test_key"
    local value="test_value_$(date +%s)"
    
    echo "Writing to master ($master_pod): $key = $value"
    kubectl exec -it $master_pod -c redis -- redis-cli SET $key "$value"
    
    echo "Reading from slave ($slave_pod)..."
    local attempt=1
    local max_attempts=30
    while [ $attempt -le $max_attempts ]; do
        read_value=$(kubectl exec -it $slave_pod -c redis -- redis-cli GET $key)
        echo "Attempt $attempt: Raw read value from slave: '$read_value'"
        
        # Strip first and last character
        read_value=${read_value:1:-2}
        
        echo "Obtained value: '$read_value', Expected value: '$value'"
        
        if [ "$read_value" = "$value" ]; then
            echo "Successfully read correct value from slave on attempt $attempt"
            return 0
        else
            echo "Attempt $attempt: Values do not match. Waiting for replication..."
        fi
        attempt=$((attempt+1))
        sleep 1
    done
    echo "Failed to read correct value from slave after $max_attempts attempts"
    return 1
}

# Get Redis pods
redis_pods=$(kubectl get pods | grep edr-router-redis | awk '{print $1}')

# Initialize variables
master_pod=""
slave_pod=""
master_ip=""
slave_ip=""

# Identify master and slave pods
for pod in $redis_pods; do
    pod_info=$(get_pod_info $pod)
    role=$(echo $pod_info | awk '{print $2}')
    ip=$(echo $pod_info | awk '{print $3}')
    
    if [ "$role" = "master" ]; then
        master_pod=$pod
        master_ip=$ip
    elif [ "$role" = "slave" ]; then
        slave_pod=$pod
        slave_ip=$ip
    fi
done

echo "Current Master: $master_pod ($master_ip)"
echo "Current Slave: $slave_pod ($slave_ip)"

# Perform failover
echo "Initiating failover..."

# Make slave the new master
echo "Promoting slave to master..."
kubectl exec -it $slave_pod -c redis -- redis-cli SLAVEOF NO ONE

# Make old master the new slave
echo "Demoting old master to slave..."
kubectl exec -it $master_pod -c redis -- redis-cli SLAVEOF $slave_ip 6379

echo "Failover initiated."

# Monitor slave promotion
monitor_slave_promotion $slave_pod

# Monitor new slave synchronization
monitor_slave_sync $master_pod $slave_ip

# Verify new roles
echo "Verifying new roles..."
for pod in $redis_pods; do
    pod_info=$(get_pod_info $pod)
    echo $pod_info
done

# Check replication info
echo "Checking replication info..."
for pod in $redis_pods; do
    check_replication_info $pod
done

# Write to new master and read from new slave
echo "Testing write to new master and read from new slave..."
if write_and_read $slave_pod $master_pod; then
    echo "Write and read test completed successfully."
else
    echo "Write and read test failed. Please check Redis replication status."
fi

echo "Failover, verification, and write/read test completed."