Last active
October 17, 2025 08:05
-
-
Save j9t/269c2181124082ff05ab6dad9baa2b49 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # Optimized IP Country Block Generator for .htaccess | |
| # (Make executable: chmod +x generate-country-blocks.sh) | |
| # Usage: ./generate-country-blocks.sh [OPTIONS] [country1] [country2] … | |
| set -e | |
| # Configuration | |
| OUTPUT_FILE="generate-country-blocks.htaccess" | |
| TEMP_DIR=$(mktemp -d) | |
| EXCLUDE_SMALL=false | |
| MIN_PREFIX_LENGTH=32 | |
| VERBOSE=false | |
| # Colors for output | |
| RED='\033[0;31m' | |
| GREEN='\033[0;32m' | |
| YELLOW='\033[1;33m' | |
| BLUE='\033[0;34m' | |
| NC='\033[0m' # No color | |
| # Statistics tracking (using simple variables for compatibility) | |
| TOTAL_BEFORE=0 | |
| TOTAL_AFTER=0 | |
| COUNTRY_STATS="" | |
| # Function to show usage | |
| show_usage() { | |
| echo "Usage: $0 [OPTIONS] [country1] [country2] …" | |
| echo "" | |
| echo "Options:" | |
| echo " --exclude-small Exclude /31 and /32 ranges for better performance (same as “--min-prefix 30”)" | |
| echo " --min-prefix N Only include ranges with prefix length ≤ N (default: 32)" | |
| echo " --verbose, -v Show detailed processing information" | |
| echo " --help, -h Show this help message" | |
| echo "" | |
| echo "Examples:" | |
| echo " $0 ru il # Russia and Israel" | |
| echo " $0 hu # Hungary only" | |
| echo " $0 --exclude-small ru il cn # Multiple countries, exclude small ranges" | |
| echo " $0 --min-prefix 24 ru # Only ranges /24 and larger" | |
| echo "" | |
| } | |
| # Parse command line arguments | |
| COUNTRIES=() | |
| while [[ $# -gt 0 ]]; do | |
| case $1 in | |
| --exclude-small) | |
| EXCLUDE_SMALL=true | |
| MIN_PREFIX_LENGTH=30 | |
| shift | |
| ;; | |
| --min-prefix) | |
| MIN_PREFIX_LENGTH="$2" | |
| if ! [[ "$MIN_PREFIX_LENGTH" =~ ^[0-9]+$ ]] || [ "$MIN_PREFIX_LENGTH" -lt 1 ] || [ "$MIN_PREFIX_LENGTH" -gt 32 ]; then | |
| echo -e "${RED}Error: “--min-prefix” must be a number between “1” and “32”${NC}" >&2 | |
| exit 1 | |
| fi | |
| shift 2 | |
| ;; | |
| --verbose|-v) | |
| VERBOSE=true | |
| shift | |
| ;; | |
| --help|-h) | |
| show_usage | |
| exit 0 | |
| ;; | |
| --*) | |
| echo -e "${RED}Error: Unknown option $1${NC}" >&2 | |
| show_usage | |
| exit 1 | |
| ;; | |
| *) | |
| COUNTRIES+=("$1") | |
| shift | |
| ;; | |
| esac | |
| done | |
| # Require at least one country to be specified | |
| if [ ${#COUNTRIES[@]} -eq 0 ]; then | |
| echo -e "${RED}Error: At least one country code must be specified${NC}" >&2 | |
| show_usage | |
| exit 1 | |
| fi | |
| # Logging functions | |
| log_info() { | |
| echo -e "${BLUE}ℹ️ $1${NC}" >&2 | |
| } | |
| log_success() { | |
| echo -e "${GREEN}✅ $1${NC}" >&2 | |
| } | |
| log_warning() { | |
| echo -e "${YELLOW}⚠️ $1${NC}" >&2 | |
| } | |
| log_error() { | |
| echo -e "${RED}❌ $1${NC}" >&2 | |
| } | |
| log_verbose() { | |
| if [ "$VERBOSE" = true ]; then | |
| echo -e "${NC}🔍 $1${NC}" >&2 | |
| fi | |
| } | |
| # Python script for IP aggregation and optimization | |
| create_ip_optimizer() { | |
| cat > "$TEMP_DIR/optimize_ips.py" << 'EOF' | |
| #!/usr/bin/env python3 | |
| import sys | |
| import ipaddress | |
| from collections import defaultdict | |
| import argparse | |
| def optimize_ip_ranges(input_file, min_prefix_length=32): | |
| """ | |
| Optimize IP ranges by: | |
| 1. Parsing and validating all ranges | |
| 2. Removing duplicates | |
| 3. Aggregating overlapping/adjacent ranges | |
| 4. Sorting by network size (larger first) | |
| 5. Filtering by minimum prefix length | |
| """ | |
| networks = [] | |
| invalid_count = 0 | |
| duplicate_count = 0 | |
| # Read and parse IP ranges | |
| seen_networks = set() | |
| try: | |
| with open(input_file, 'r') as f: | |
| for line_num, line in enumerate(f, 1): | |
| line = line.strip() | |
| if not line or line.startswith('#'): | |
| continue | |
| try: | |
| network = ipaddress.IPv4Network(line, strict=False) | |
| # Filter by minimum prefix length | |
| if network.prefixlen > min_prefix_length: | |
| continue | |
| network_str = str(network) | |
| if network_str in seen_networks: | |
| duplicate_count += 1 | |
| continue | |
| seen_networks.add(network_str) | |
| networks.append(network) | |
| except (ipaddress.AddressValueError, ValueError) as e: | |
| invalid_count += 1 | |
| print(f"Warning: Invalid IP range on line {line_num}: {line}", file=sys.stderr) | |
| except FileNotFoundError: | |
| print(f"Error: Cannot read file {input_file}", file=sys.stderr) | |
| return [], {}, invalid_count, duplicate_count | |
| original_count = len(networks) | |
| # Aggregate overlapping and adjacent networks | |
| if networks: | |
| try: | |
| # Sort networks for better aggregation | |
| networks.sort(key=lambda x: (x.network_address, x.prefixlen)) | |
| aggregated = list(ipaddress.collapse_addresses(networks)) | |
| # Sort by network size (larger blocks first) for better Apache performance | |
| aggregated.sort(key=lambda x: (x.prefixlen, x.network_address)) | |
| except Exception as e: | |
| print(f"Warning: Aggregation failed, using original networks: {e}", file=sys.stderr) | |
| aggregated = networks | |
| else: | |
| aggregated = [] | |
| stats = { | |
| 'original_count': original_count + duplicate_count, | |
| 'after_dedup': original_count, | |
| 'final_count': len(aggregated), | |
| 'invalid_count': invalid_count, | |
| 'duplicate_count': duplicate_count, | |
| 'reduction_percent': round((1 - len(aggregated) / max(1, original_count + duplicate_count)) * 100, 1) if original_count + duplicate_count > 0 else 0 | |
| } | |
| return aggregated, stats, invalid_count, duplicate_count | |
| def main(): | |
| parser = argparse.ArgumentParser(description='Optimize IP address ranges') | |
| parser.add_argument('input_file', help='Input file containing IP ranges') | |
| parser.add_argument('--min-prefix', type=int, default=32, help='Minimum prefix length to include') | |
| parser.add_argument('--stats-only', action='store_true', help='Only output statistics') | |
| args = parser.parse_args() | |
| networks, stats, invalid_count, duplicate_count = optimize_ip_ranges(args.input_file, args.min_prefix) | |
| if args.stats_only: | |
| print(f"STATS:{stats['original_count']}:{stats['final_count']}:{stats['reduction_percent']}:{invalid_count}:{duplicate_count}") | |
| else: | |
| for network in networks: | |
| print(str(network)) | |
| if __name__ == '__main__': | |
| main() | |
| EOF | |
| chmod +x "$TEMP_DIR/optimize_ips.py" | |
| } | |
| # Function to download and process country data with optimization | |
| process_country() { | |
| local country=$1 | |
| local country_upper=$(echo "$country" | tr '[:lower:]' '[:upper:]') | |
| local zone_file="$TEMP_DIR/${country}.zone" | |
| local optimized_file="$TEMP_DIR/${country}.optimized" | |
| log_info "Downloading $country_upper IP ranges…" | |
| # Download country zone file with better error handling | |
| local download_success=false | |
| local retry_count=0 | |
| local max_retries=3 | |
| while [ $retry_count -lt $max_retries ] && [ "$download_success" = false ]; do | |
| if curl -s -f --connect-timeout 30 --max-time 60 \ | |
| "http://www.ipdeny.com/ipblocks/data/countries/${country}.zone" > "$zone_file" 2>/dev/null; then | |
| # Verify download contains data | |
| if [ -s "$zone_file" ] && grep -q "^[0-9]" "$zone_file"; then | |
| download_success=true | |
| local range_count=$(wc -l < "$zone_file") | |
| TOTAL_BEFORE=$((TOTAL_BEFORE + range_count)) | |
| log_success "Downloaded $range_count IP ranges for $country_upper" | |
| else | |
| log_warning "Downloaded file for $country_upper is empty or invalid" | |
| rm -f "$zone_file" | |
| fi | |
| else | |
| retry_count=$((retry_count + 1)) | |
| if [ $retry_count -lt $max_retries ]; then | |
| log_warning "Download attempt $retry_count failed for $country_upper, retrying…" | |
| sleep 2 | |
| fi | |
| fi | |
| done | |
| if [ "$download_success" = false ]; then | |
| log_error "Failed to download data for $country_upper after $max_retries attempts" | |
| echo " # ERROR: Could not download $country_upper ranges (failed after $max_retries attempts)" | |
| return 1 | |
| fi | |
| # Optimize IP ranges | |
| log_verbose "Optimizing IP ranges for $country_upper…" | |
| if python3 "$TEMP_DIR/optimize_ips.py" "$zone_file" --min-prefix "$MIN_PREFIX_LENGTH" > "$optimized_file" 2>/dev/null; then | |
| # Get optimization stats | |
| local stats_output=$(python3 "$TEMP_DIR/optimize_ips.py" "$zone_file" --min-prefix "$MIN_PREFIX_LENGTH" --stats-only 2>/dev/null) | |
| local original_count=$(echo "$stats_output" | cut -d':' -f2) | |
| local final_count=$(echo "$stats_output" | cut -d':' -f3) | |
| local reduction_percent=$(echo "$stats_output" | cut -d':' -f4) | |
| local invalid_count=$(echo "$stats_output" | cut -d':' -f5) | |
| local duplicate_count=$(echo "$stats_output" | cut -d':' -f6) | |
| TOTAL_AFTER=$((TOTAL_AFTER + final_count)) | |
| log_success "Optimized $country_upper: $original_count → $final_count ranges ($reduction_percent% reduction)" | |
| if [ $invalid_count -gt 0 ]; then | |
| log_warning "$country_upper had $invalid_count invalid IP ranges" | |
| fi | |
| if [ $duplicate_count -gt 0 ]; then | |
| log_verbose "$country_upper had $duplicate_count duplicate ranges removed" | |
| fi | |
| # Generate output with enhanced comments | |
| echo " # $country_upper IP ranges" | |
| while IFS= read -r ip_range; do | |
| if [[ -n "$ip_range" ]]; then | |
| echo " Require not ip $ip_range" | |
| fi | |
| done < "$optimized_file" | |
| else | |
| log_error "Failed to optimize IP ranges for $country_upper, using original data" | |
| echo " # $country_upper IP ranges (optimization failed, using original)" | |
| local line_count=0 | |
| while IFS= read -r ip_range; do | |
| if [[ -n "$ip_range" && ! "$ip_range" =~ ^# ]]; then | |
| # Apply minimum prefix filter manually if optimization failed | |
| if [ "$MIN_PREFIX_LENGTH" -lt 32 ]; then | |
| local prefix=$(echo "$ip_range" | cut -d'/' -f2) | |
| if [ "$prefix" -gt "$MIN_PREFIX_LENGTH" ]; then | |
| continue | |
| fi | |
| fi | |
| echo " Require not ip $ip_range" | |
| line_count=$((line_count + 1)) | |
| fi | |
| done < "$zone_file" | |
| TOTAL_AFTER=$((TOTAL_AFTER + line_count)) | |
| fi | |
| } | |
| # Main script execution | |
| log_info "🌍 Generating optimized .htaccess country blocks" | |
| log_info "Countries: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]')" | |
| log_info "Output file: $OUTPUT_FILE" | |
| if [ "$EXCLUDE_SMALL" = true ]; then | |
| log_info "Excluding small ranges (larger than /$MIN_PREFIX_LENGTH)" | |
| else | |
| log_info "Including all ranges up to /$MIN_PREFIX_LENGTH" | |
| fi | |
| log_verbose "Using temp directory: $TEMP_DIR" | |
| # Create Python optimizer | |
| create_ip_optimizer | |
| # Check if Python 3 is available | |
| if ! command -v python3 &> /dev/null; then | |
| log_error "Python 3 is required for IP optimization but is not installed" | |
| log_error "Please install Python 3 or use the original script for basic functionality" | |
| exit 1 | |
| fi | |
| # Generate .htaccess header with enhanced metadata | |
| { | |
| generation_date=$(date '+%Y-%m-%d %H:%M:%S %Z') | |
| echo "# Country IP Blocks" | |
| echo "# Generated: $generation_date" | |
| echo "# Source: IPDeny.com" | |
| echo "# Exceptions" | |
| echo '<FilesMatch "^(error403\.html)$">' | |
| echo " Require all granted" | |
| echo "</FilesMatch>" | |
| if [ "$MIN_PREFIX_LENGTH" -lt 32 ]; then | |
| echo "# Countries: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]') (only ranges /$MIN_PREFIX_LENGTH and larger included)" | |
| else | |
| echo "# Countries: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]')" | |
| fi | |
| echo "<RequireAll>" | |
| echo " Require all granted" | |
| # Process each country | |
| failed_countries="" | |
| for country in "${COUNTRIES[@]}"; do | |
| if ! process_country "$country"; then | |
| failed_countries="$failed_countries $country" | |
| fi | |
| done | |
| echo "</RequireAll>" | |
| } > "$OUTPUT_FILE" | |
| # Clean up | |
| rm -rf "$TEMP_DIR" | |
| # Display results and statistics | |
| echo "" | |
| log_success "🎉 Generated $OUTPUT_FILE" | |
| # File size and range count | |
| final_ranges=$(grep -c "Require not ip" "$OUTPUT_FILE" || echo "0") | |
| file_size=$(du -h "$OUTPUT_FILE" | cut -f1) | |
| echo "" | |
| echo "📊 Optimization results:" | |
| echo " Before: $TOTAL_BEFORE IP ranges" | |
| echo " After: $TOTAL_AFTER IP ranges" | |
| if [ $TOTAL_BEFORE -gt 0 ]; then | |
| reduction_percent=$(( (TOTAL_BEFORE - TOTAL_AFTER) * 100 / TOTAL_BEFORE )) | |
| echo " Reduction: $reduction_percent% ($(( TOTAL_BEFORE - TOTAL_AFTER )) ranges removed)" | |
| fi | |
| echo " File size: $file_size" | |
| echo "" | |
| echo "📋 Countries processed: $(echo "${COUNTRIES[*]}" | tr '[:lower:]' '[:upper:]')" | |
| echo "" | |
| echo "📄 To use:" | |
| echo " cat $OUTPUT_FILE" | |
| echo " # Copy the “<RequireAll>” block to your .htaccess file" | |
| echo "" | |
| # Performance recommendations | |
| if [ $TOTAL_AFTER -gt 1000 ]; then | |
| log_warning "Large number of ranges ($TOTAL_AFTER) may impact Apache performance" | |
| echo " 💡 Consider using “--exclude-small” or “--min-prefix 24” for better performance" | |
| elif [ $TOTAL_AFTER -gt 500 ]; then | |
| echo " 💡 Range count ($TOTAL_AFTER) is moderate—should perform well" | |
| else | |
| echo " 💡 Excellent optimization! Only $TOTAL_AFTER ranges for optimal performance" | |
| fi | |
| log_success "Generation completed: $(date '+%Y-%m-%d %H:%M:%S')" |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@nisbet-hubbard, thanks! I just updated the script so to make error403.html pages an exception, one that is also configurable (other files could be made exempt, too). From my tests, this works well.