Get TCP ephemeral port details to validate ephemeral port exhaustion.

ephemeral-port-details.sh

#!/bin/bash
#
# REQUIRES
# - netstat - to retrieve network stack details
# - sysctl  - to retrieve/modify kernel settings
# - bc      - for math
#
# MIT License
#
# Copyright 2020 John H Patton, JH Patton Consulting, LLC
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 
# OTHER DEALINGS IN THE SOFTWARE.

# Interactive or not?
[ -t "0" ] && INTERACTIVE=1 || INTERACTIVE=0

# If interactive, set output colors:
if (( INTERACTIVE )); then
  # CONSTANTS
  # foreground colors, use with echo -e
  NC='\033[0m'       # No Color, reset
  # normal;            bold;                 high intensity;       bold + high intensity
  BLACK='\033[0;30m';  BBLACK='\033[1;30m';  IBLACK='\033[0;90m';  BIBLACK='\033[1;90m';
  RED='\033[0;31m';    BRED='\033[1;31m';    IRED='\033[0;91m';    BIRED='\033[1;91m';
  GREEN='\033[0;32m';  BGREEN='\033[1;32m';  IGREEN='\033[0;92m';  BIGREEN='\033[1;92m';
  YELLOW='\033[0;33m'; BYELLOW='\033[1;33m'; IYELLOW='\033[0;93m'; BIYELLOW='\033[1;93m';
  BLUE='\033[0;34m';   BBLUE='\033[1;34m';   IBLUE='\033[0;94m';   BIBLUE='\033[1;94m';
  PURPLE='\033[0;35m'; BPURPLE='\033[1;35m'; IPURPLE='\033[0;95m'; BIPURPLE='\033[1;95m';
  CYAN='\033[0;36m';   BCYAN='\033[1;36m';   ICYAN='\033[0;96m';   BICYAN='\033[1;96m';
  WHITE='\033[0;37m';  BWHITE='\033[1;37m';  IWHITE='\033[0;97m';  BIWHITE='\033[1;97m';
fi

# loggers
# adjust colors for terminal display needs
loginfo() { logmsg "${BICYAN}INFO: ${NC}$1${NC}";     }
logok()   { logmsg "${BIGREEN}  OK: ${NC}$1${NC}";    }
logwarn() { logmsg "${BIYELLOW}WARN: ${NC}$1${NC}";  }
logcrit() { logmsg "${BIRED}CRIT: ${NC}$1${NC}";       }
logmsg()  { [[ ${FUNCNAME[1]} =~ log.* ]] && echo -e "$1" || loginfo "$1"; }

# Get port range
range=$(sysctl net.ipv4.ip_local_port_range)
low_port=$(echo $range | awk '{print $3}')
high_port=$(echo $range | awk '{print $4}')
total=$((high_port - low_port))

# Set port thresholds
WARN_PERCENTAGE=70
CRIT_PERCENTAGE=85

ephemeral_tw_count=0
ephemeral_port_count=0
while read line; do
  # Get ip:port combo and extract port
  local_socket=$(echo $line | awk '{print $4}')
  port=${local_socket##*:}
  
  if ((port >= low_port)) && ((port <= high_port)); then
    # Port in local socket is in the ephemeral port range
    ephemeral_port_count=$((ephemeral_port_count+1))
    if [[ $line =~ TIME_WAIT ]]; then
      ephemeral_tw_count=$((ephemeral_tw_count+1))
    fi
  fi
done< <(netstat -n -t -4 | grep -F "$(hostname -i)")

unused_ephemeral_ports=$(( total - ephemeral_port_count))

# Used Percentage
used_calc="scale=2; (${ephemeral_port_count}/${total}) * 100"
used_percent=$(echo "$used_calc" | bc)
used_percent="${used_percent%.*}"

# Time Wait Percentage
tw_calc="scale=2; (${ephemeral_tw_count}/${ephemeral_port_count}) * 100"
tw_percent="0"
(( ephemeral_port_count > 0 )) && tw_percent=$(echo "$tw_calc" | bc)
tw_percent="${tw_percent%.*}"


loginfo "Ephemeral port range: ${BGREEN}${low_port} - ${high_port}"
loginfo "Total ephemeral ports for client connections: ${BGREEN}${total}"
loginfo "Count of active ephemeral ports: ${BGREEN}${ephemeral_port_count}"
loginfo "Unused ephemeral ports: ${BGREEN}${unused_ephemeral_ports}"

if ((used_percent > CRIT_PERCENTAGE)); then
    percent_color="${BIRED}"
    logcrit "Running very low on ephemeral ports! Consider increasing: net.ipv4.ip_local_port_range"
elif ((used_percent > WARN_PERCENTAGE)); then
    percent_color="${BIYELLOW}"
    logwarn "The system is using a high volume of ephemeral ports."
else
    percent_color="${BIGREEN}"
    logok "The system ephemeral port usage is healthy."
fi
loginfo "Percent of active ephemeral ports in use: ${percent_color}${used_percent}%"

if ((tw_percent > CRIT_PERCENTAGE)); then
    percent_color="${BIRED}"
    logcrit "High percentage of client connections are in TIME_WAIT, check outbound connectivity."
else
    percent_color="${BIGREEN}"
fi
loginfo "Percent of active ephemeral ports in TIME_WAIT: ${percent_color}${tw_percent}%"