Skip to content

Instantly share code, notes, and snippets.

@j9t
Last active October 17, 2025 08:05
Show Gist options
  • Select an option

  • Save j9t/269c2181124082ff05ab6dad9baa2b49 to your computer and use it in GitHub Desktop.

Select an option

Save j9t/269c2181124082ff05ab6dad9baa2b49 to your computer and use it in GitHub Desktop.
#!/bin/bash
# Optimized IP Country Block Generator for .htaccess
# (Make executable: chmod +x generate-country-blocks.sh)
# Usage: ./generate-country-blocks.sh [OPTIONS] [country1] [country2] …
set -e
# Configuration
OUTPUT_FILE="generate-country-blocks.htaccess"
TEMP_DIR=$(mktemp -d)
EXCLUDE_SMALL=false
MIN_PREFIX_LENGTH=32
VERBOSE=false
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No color
# Statistics tracking (using simple variables for compatibility)
TOTAL_BEFORE=0
TOTAL_AFTER=0
COUNTRY_STATS=""
# Function to show usage
show_usage() {
echo "Usage: $0 [OPTIONS] [country1] [country2] …"
echo ""
echo "Options:"
echo " --exclude-small Exclude /31 and /32 ranges for better performance (same as “--min-prefix 30”)"
echo " --min-prefix N Only include ranges with prefix length ≤ N (default: 32)"
echo " --verbose, -v Show detailed processing information"
echo " --help, -h Show this help message"
echo ""
echo "Examples:"
echo " $0 ru il # Russia and Israel"
echo " $0 hu # Hungary only"
echo " $0 --exclude-small ru il cn # Multiple countries, exclude small ranges"
echo " $0 --min-prefix 24 ru # Only ranges /24 and larger"
echo ""
}
# Parse command line arguments
COUNTRIES=()
while [[ $# -gt 0 ]]; do
case $1 in
--exclude-small)
EXCLUDE_SMALL=true
MIN_PREFIX_LENGTH=30
shift
;;
--min-prefix)
MIN_PREFIX_LENGTH="$2"
if ! [[ "$MIN_PREFIX_LENGTH" =~ ^[0-9]+$ ]] || [ "$MIN_PREFIX_LENGTH" -lt 1 ] || [ "$MIN_PREFIX_LENGTH" -gt 32 ]; then
echo -e "${RED}Error: “--min-prefix” must be a number between “1” and “32”${NC}" >&2
exit 1
fi
shift 2
;;
--verbose|-v)
VERBOSE=true
shift
;;
--help|-h)
show_usage
exit 0
;;
--*)
echo -e "${RED}Error: Unknown option $1${NC}" >&2
show_usage
exit 1
;;
*)
COUNTRIES+=("$1")
shift
;;
esac
done
# Require at least one country to be specified
if [ ${#COUNTRIES[@]} -eq 0 ]; then
echo -e "${RED}Error: At least one country code must be specified${NC}" >&2
show_usage
exit 1
fi
# Logging functions
log_info() {
echo -e "${BLUE}ℹ️ $1${NC}" >&2
}
log_success() {
echo -e "${GREEN}✅ $1${NC}" >&2
}
log_warning() {
echo -e "${YELLOW}⚠️ $1${NC}" >&2
}
log_error() {
echo -e "${RED}❌ $1${NC}" >&2
}
log_verbose() {
if [ "$VERBOSE" = true ]; then
echo -e "${NC}🔍 $1${NC}" >&2
fi
}
# Python script for IP aggregation and optimization
create_ip_optimizer() {
cat > "$TEMP_DIR/optimize_ips.py" << 'EOF'
#!/usr/bin/env python3
import sys
import ipaddress
from collections import defaultdict
import argparse
def optimize_ip_ranges(input_file, min_prefix_length=32):
"""
Optimize IP ranges by:
1. Parsing and validating all ranges
2. Removing duplicates
3. Aggregating overlapping/adjacent ranges
4. Sorting by network size (larger first)
5. Filtering by minimum prefix length
"""
networks = []
invalid_count = 0
duplicate_count = 0
# Read and parse IP ranges
seen_networks = set()
try:
with open(input_file, 'r') as f:
for line_num, line in enumerate(f, 1):
line = line.strip()
if not line or line.startswith('#'):
continue
try:
network = ipaddress.IPv4Network(line, strict=False)
# Filter by minimum prefix length
if network.prefixlen > min_prefix_length:
continue
network_str = str(network)
if network_str in seen_networks:
duplicate_count += 1
continue
seen_networks.add(network_str)
networks.append(network)
except (ipaddress.AddressValueError, ValueError) as e:
invalid_count += 1
print(f"Warning: Invalid IP range on line {line_num}: {line}", file=sys.stderr)
except FileNotFoundError:
print(f"Error: Cannot read file {input_file}", file=sys.stderr)
return [], {}, invalid_count, duplicate_count
original_count = len(networks)
# Aggregate overlapping and adjacent networks
if networks:
try:
# Sort networks for better aggregation
networks.sort(key=lambda x: (x.network_address, x.prefixlen))
aggregated = list(ipaddress.collapse_addresses(networks))
# Sort by network size (larger blocks first) for better Apache performance
aggregated.sort(key=lambda x: (x.prefixlen, x.network_address))
except Exception as e:
print(f"Warning: Aggregation failed, using original networks: {e}", file=sys.stderr)
aggregated = networks
else:
aggregated = []
stats = {
'original_count': original_count + duplicate_count,
'after_dedup': original_count,
'final_count': len(aggregated),
'invalid_count': invalid_count,
'duplicate_count': duplicate_count,
'reduction_percent': round((1 - len(aggregated) / max(1, original_count + duplicate_count)) * 100, 1) if original_count + duplicate_count > 0 else 0
}
return aggregated, stats, invalid_count, duplicate_count
def main():
parser = argparse.ArgumentParser(description='Optimize IP address ranges')
parser.add_argument('input_file', help='Input file containing IP ranges')
parser.add_argument('--min-prefix', type=int, default=32, help='Minimum prefix length to include')
parser.add_argument('--stats-only', action='store_true', help='Only output statistics')
args = parser.parse_args()
networks, stats, invalid_count, duplicate_count = optimize_ip_ranges(args.input_file, args.min_prefix)
if args.stats_only:
print(f"STATS:{stats['original_count']}:{stats['final_count']}:{stats['reduction_percent']}:{invalid_count}:{duplicate_count}")
else:
for network in networks:
print(str(network))
if __name__ == '__main__':
main()
EOF
chmod +x "$TEMP_DIR/optimize_ips.py"
}
# Function to download and process country data with optimization
process_country() {
local country=$1
local country_upper=$(echo "$country" | tr '[:lower:]' '[:upper:]')
local zone_file="$TEMP_DIR/${country}.zone"
local optimized_file="$TEMP_DIR/${country}.optimized"
log_info "Downloading $country_upper IP ranges…"
# Download country zone file with better error handling
local download_success=false
local retry_count=0
local max_retries=3
while [ $retry_count -lt $max_retries ] && [ "$download_success" = false ]; do
if curl -s -f --connect-timeout 30 --max-time 60 \
"http://www.ipdeny.com/ipblocks/data/countries/${country}.zone" > "$zone_file" 2>/dev/null; then
# Verify download contains data
if [ -s "$zone_file" ] && grep -q "^[0-9]" "$zone_file"; then
download_success=true
local range_count=$(wc -l < "$zone_file")
TOTAL_BEFORE=$((TOTAL_BEFORE + range_count))
log_success "Downloaded $range_count IP ranges for $country_upper"
else
log_warning "Downloaded file for $country_upper is empty or invalid"
rm -f "$zone_file"
fi
else
retry_count=$((retry_count + 1))
if [ $retry_count -lt $max_retries ]; then
log_warning "Download attempt $retry_count failed for $country_upper, retrying…"
sleep 2
fi
fi
done
if [ "$download_success" = false ]; then
log_error "Failed to download data for $country_upper after $max_retries attempts"
echo " # ERROR: Could not download $country_upper ranges (failed after $max_retries attempts)"
return 1
fi
# Optimize IP ranges
log_verbose "Optimizing IP ranges for $country_upper…"
if python3 "$TEMP_DIR/optimize_ips.py" "$zone_file" --min-prefix "$MIN_PREFIX_LENGTH" > "$optimized_file" 2>/dev/null; then
# Get optimization stats
local stats_output=$(python3 "$TEMP_DIR/optimize_ips.py" "$zone_file" --min-prefix "$MIN_PREFIX_LENGTH" --stats-only 2>/dev/null)
local original_count=$(echo "$stats_output" | cut -d':' -f2)
local final_count=$(echo "$stats_output" | cut -d':' -f3)
local reduction_percent=$(echo "$stats_output" | cut -d':' -f4)
local invalid_count=$(echo "$stats_output" | cut -d':' -f5)
local duplicate_count=$(echo "$stats_output" | cut -d':' -f6)
TOTAL_AFTER=$((TOTAL_AFTER + final_count))
log_success "Optimized $country_upper: $original_count → $final_count ranges ($reduction_percent% reduction)"
if [ $invalid_count -gt 0 ]; then
log_warning "$country_upper had $invalid_count invalid IP ranges"
fi
if [ $duplicate_count -gt 0 ]; then
log_verbose "$country_upper had $duplicate_count duplicate ranges removed"
fi
# Generate output with enhanced comments
echo " # $country_upper IP ranges"
while IFS= read -r ip_range; do
if [[ -n "$ip_range" ]]; then
echo " Require not ip $ip_range"
fi
done < "$optimized_file"
else
log_error "Failed to optimize IP ranges for $country_upper, using original data"
echo " # $country_upper IP ranges (optimization failed, using original)"
local line_count=0
while IFS= read -r ip_range; do
if [[ -n "$ip_range" && ! "$ip_range" =~ ^# ]]; then
# Apply minimum prefix filter manually if optimization failed
if [ "$MIN_PREFIX_LENGTH" -lt 32 ]; then
local prefix=$(echo "$ip_range" | cut -d'/' -f2)
if [ "$prefix" -gt "$MIN_PREFIX_LENGTH" ]; then
continue
fi
fi
echo " Require not ip $ip_range"
line_count=$((line_count + 1))
fi
done < "$zone_file"
TOTAL_AFTER=$((TOTAL_AFTER + line_count))
fi
}
# Main script execution
log_info "🌍 Generating optimized .htaccess country blocks"
log_info "Countries: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]')"
log_info "Output file: $OUTPUT_FILE"
if [ "$EXCLUDE_SMALL" = true ]; then
log_info "Excluding small ranges (larger than /$MIN_PREFIX_LENGTH)"
else
log_info "Including all ranges up to /$MIN_PREFIX_LENGTH"
fi
log_verbose "Using temp directory: $TEMP_DIR"
# Create Python optimizer
create_ip_optimizer
# Check if Python 3 is available
if ! command -v python3 &> /dev/null; then
log_error "Python 3 is required for IP optimization but is not installed"
log_error "Please install Python 3 or use the original script for basic functionality"
exit 1
fi
# Generate .htaccess header with enhanced metadata
{
generation_date=$(date '+%Y-%m-%d %H:%M:%S %Z')
echo "# Country IP Blocks"
echo "# Generated: $generation_date"
echo "# Source: IPDeny.com"
echo "# Exceptions"
echo '<FilesMatch "^(error403\.html)$">'
echo " Require all granted"
echo "</FilesMatch>"
if [ "$MIN_PREFIX_LENGTH" -lt 32 ]; then
echo "# Countries: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]') (only ranges /$MIN_PREFIX_LENGTH and larger included)"
else
echo "# Countries: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]')"
fi
echo "<RequireAll>"
echo " Require all granted"
# Process each country
failed_countries=""
for country in "${COUNTRIES[@]}"; do
if ! process_country "$country"; then
failed_countries="$failed_countries $country"
fi
done
echo "</RequireAll>"
} > "$OUTPUT_FILE"
# Clean up
rm -rf "$TEMP_DIR"
# Display results and statistics
echo ""
log_success "🎉 Generated $OUTPUT_FILE"
# File size and range count
final_ranges=$(grep -c "Require not ip" "$OUTPUT_FILE" || echo "0")
file_size=$(du -h "$OUTPUT_FILE" | cut -f1)
echo ""
echo "📊 Optimization results:"
echo " Before: $TOTAL_BEFORE IP ranges"
echo " After: $TOTAL_AFTER IP ranges"
if [ $TOTAL_BEFORE -gt 0 ]; then
reduction_percent=$(( (TOTAL_BEFORE - TOTAL_AFTER) * 100 / TOTAL_BEFORE ))
echo " Reduction: $reduction_percent% ($(( TOTAL_BEFORE - TOTAL_AFTER )) ranges removed)"
fi
echo " File size: $file_size"
echo ""
echo "📋 Countries processed: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]')"
echo ""
echo "📄 To use:"
echo " cat $OUTPUT_FILE"
echo " # Copy the “<RequireAll>” block to your .htaccess file"
echo ""
# Performance recommendations
if [ $TOTAL_AFTER -gt 1000 ]; then
log_warning "Large number of ranges ($TOTAL_AFTER) may impact Apache performance"
echo " 💡 Consider using “--exclude-small” or “--min-prefix 24” for better performance"
elif [ $TOTAL_AFTER -gt 500 ]; then
echo " 💡 Range count ($TOTAL_AFTER) is moderate—should perform well"
else
echo " 💡 Excellent optimization! Only $TOTAL_AFTER ranges for optimal performance"
fi
log_success "Generation completed: $(date '+%Y-%m-%d %H:%M:%S')"
@j9t
Copy link
Author

j9t commented Oct 17, 2025

@nisbet-hubbard, thanks! I just updated the script so to make error403.html pages an exception, one that is also configurable (other files could be made exempt, too). From my tests, this works well.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment