check-vals.sh

#!/bin/bash

# Array of validator endpoints
validators=(
    # Paris validators
    "conval-01.par.mammochain.com"
    "conval-02.par.mammochain.com"
    "conval-03.par.mammochain.com"
    "conval-04.par.mammochain.com"
    "conval-05.par.mammochain.com"
    "conval-06.par.mammochain.com"
    "conval-07.par.mammochain.com"
    # Amsterdam validators
    "conval-08.ams.mammochain.com"
    "conval-09.ams.mammochain.com"
    "conval-10.ams.mammochain.com"
    "conval-11.ams.mammochain.com"
    "conval-12.ams.mammochain.com"
    "conval-13.ams.mammochain.com"
    "conval-14.ams.mammochain.com"
    # Warsaw validators
    "conval-15.waw.mammochain.com"
    "conval-16.waw.mammochain.com"
    "conval-17.waw.mammochain.com"
    "conval-18.waw.mammochain.com"
    "conval-19.waw.mammochain.com"
    "conval-20.waw.mammochain.com"
    "conval-21.waw.mammochain.com"
)

# Function to get current location based on validator hostname
get_location() {
    if [[ $1 == *".par."* ]]; then
        echo "Paris"
    elif [[ $1 == *".ams."* ]]; then
        echo "Amsterdam"
    elif [[ $1 == *".waw."* ]]; then
        echo "Warsaw"
    else
        echo "Unknown"
    fi
}

# Function to check if host is pingable
check_ping() {
    ping -c 1 -W 2 "$1" &> /dev/null
    return $?
}

# Function to get block height from validator
get_block_height() {
    local endpoint=$1
    # Try to get block height via RPC endpoint (default port 26657)
    local result=$(curl -s --connect-timeout 5 "http://${endpoint}:26657/status" | jq -r '.result.sync_info.latest_block_height' 2>/dev/null)
    
    # Check if we got a valid result
    if [[ "$result" == "null" || -z "$result" ]]; then
        echo "N/A"
    else
        echo "$result"
    fi
}

# Print header
echo "CELESTIA CONSENSUS VALIDATOR STATUS CHECK"
echo "-------------------------------------------------------------------------"
echo "Location  | Validator                       | Status  | Block Height"
echo "-------------------------------------------------------------------------"

# Initialize counters
total=0
online=0
current_location=""

# Store status and heights for later analysis
status_list=""
height_list=""

# Process validators
for validator in $(echo "${validators[@]}" | tr ' ' '\n' | sort); do
    location=$(get_location "$validator")
    
    # Check if validator is pingable
    if check_ping "$validator"; then
        ping_status="ONLINE"
        block_height=$(get_block_height "$validator")
        if [[ "$block_height" != "N/A" ]]; then
            # Store validator and its height for later analysis
            status_list="$status_list$validator:$location:$block_height\n"
            height_list="$height_list$block_height\n"
        fi
        ((online++))
    else
        ping_status="OFFLINE"
        block_height="N/A"
    fi
    
    # Print validator status with location
    printf "%-10s | %-30s | %-7s | %s\n" "$location" "$validator" "$ping_status" "$block_height"
    
    ((total++))
done

echo "-------------------------------------------------------------------------"
echo "Summary: ${online}/${total} validators online"

# Print warning if any validators are offline
if [[ $online -lt $total ]]; then
    echo "WARNING: Some validators appear to be offline or unreachable"
fi

# Process height discrepancies
echo ""
echo "Checking for block height discrepancies..."

# Get unique heights and count them
unique_heights=$(echo -e "$height_list" | sort -n | uniq)
if [[ $(echo "$unique_heights" | wc -l) -gt 1 ]]; then
    echo "WARNING: Validators are at different block heights:"
    
    # Count each height
    for height in $unique_heights; do
        if [[ -n "$height" ]]; then
            count=$(echo -e "$height_list" | grep -c "^$height$")
            echo "Block height $height: $count validators"
        fi
    done
    
    # Find the most common height
    most_common_height=$(echo -e "$height_list" | sort | uniq -c | sort -nr | head -1 | awk '{print $2}')
    
    if [[ -n "$most_common_height" ]]; then
        echo ""
        echo "Most validators are at height $most_common_height"
        echo "Validators that are behind:"
        
        # Check each validator against the most common height
        echo -e "$status_list" | while read -r line; do
            if [[ -n "$line" ]]; then
                validator=$(echo "$line" | cut -d: -f1)
                location=$(echo "$line" | cut -d: -f2)
                height=$(echo "$line" | cut -d: -f3)
                
                if [[ "$height" != "N/A" && "$height" -lt "$most_common_height" ]]; then
                    behind=$((most_common_height - height))
                    echo "- $validator ($location): $height (behind by $behind blocks)"
                fi
            fi
        done
    fi
elif [[ -n "$unique_heights" ]]; then
    echo "All online validators are at block height: $unique_heights"
else
    echo "No validators provided block height information"
fi

# Print validator statistics by location
echo ""
echo "Validator status by location:"
echo "-------------------------------------------------------------------------"
printf "%-10s | %-7s | %-7s | %s\n" "Location" "Total" "Online" "Offline"
echo "-------------------------------------------------------------------------"

# Count by location using simpler approach
paris_total=0
paris_online=0
amsterdam_total=0
amsterdam_online=0
warsaw_total=0
warsaw_online=0

for validator in "${validators[@]}"; do
    if [[ $validator == *".par."* ]]; then
        ((paris_total++))
        if check_ping "$validator"; then
            ((paris_online++))
        fi
    elif [[ $validator == *".ams."* ]]; then
        ((amsterdam_total++))
        if check_ping "$validator"; then
            ((amsterdam_online++))
        fi
    elif [[ $validator == *".waw."* ]]; then
        ((warsaw_total++))
        if check_ping "$validator"; then
            ((warsaw_online++))
        fi
    fi
done

# Print location summary
printf "%-10s | %-7s | %-7s | %s\n" "Paris" "$paris_total" "$paris_online" "$((paris_total - paris_online))"
printf "%-10s | %-7s | %-7s | %s\n" "Amsterdam" "$amsterdam_total" "$amsterdam_online" "$((amsterdam_total - amsterdam_online))"
printf "%-10s | %-7s | %-7s | %s\n" "Warsaw" "$warsaw_total" "$warsaw_online" "$((warsaw_total - warsaw_online))"
printf "%-10s | %-7s | %-7s | %s\n" "Total" "$total" "$online" "$((total - online))"
echo "-------------------------------------------------------------------------"