Quick Ruby script for extracting domains from dnsdb.org

dnsdb_crawl.rb

#!/usr/bin/env ruby

require "net/http"
require "uri"
require "cgi"
require "resolv"
require "optparse"
require "socket"
require "timeout"

LINK_REGEX   = /<a href=".*">(.*)<\/a>/

def extract_domains(uri, domain)
  domains    = []
  parsed_uri = URI.parse(uri)
  verbose("Requesting #{uri}...")
  response   = Net::HTTP.get(parsed_uri.host, parsed_uri.path)
  links      = response.scan(LINK_REGEX)
  verbose("Found #{links.count} links on page")
  return domains if links.count.zero?
  links.each do |link|
    link = link.first.strip.downcase
    if link.end_with?(domain)
      domains << link
    else
      domains += extract_domains(uri + CGI.escape(link), domain)
    end
  end
  domains
end

def port_open?(host, port)
  Timeout.timeout(0.5) do
    TCPSocket.new(host, port).close
    true
  end
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Timeout::Error
  false
end

def verbose(message)
  puts "\e[#{36}m#{message}\e[0m" if @options[:verbose]
end

@options = {
  :nameservers => ["8.8.8.8", "8.8.4.4"],
  :ports       => [],
  :deep        => false,
  :verbose     => false
}
OptionParser.new do |opts|
  opts.banner = "Usage: #{$0} [options] DOMAIN"

  opts.on("-n", "--nameservers HOSTS", "Nameservers to use (default: #{@options[:nameservers].join(',')})") do |v|
    @options[:nameservers] = v.split(",").map(&:strip)
  end

  opts.on("-p", "--ports PORTS", "Show domains with one or more of specified ports open") do |v|
    @options[:resolve] = true
    @options[:ports]   = v.split(",").map { |p| p.to_s.strip.to_i }.uniq
  end

  opts.on("-d", "--[no-]deep", "Retrieve subdomains on discovered domains") do |v|
    @options[:deep] = v
  end

  opts.on("--[no-]verbose", "Run verbosely") do |v|
    @options[:verbose] = v
  end

  if ARGV.empty?
    puts opts.help
    exit 1
  end
end.parse!

domains = extract_domains("http://www.dnsdb.org/f/#{CGI.escape(ARGV.first)}.dnsdb.org/", ARGV.first)
if @options[:deep]
  deep_domains = []
  domains.each do |domain|
    deep_domains << domain
    deep_domains += extract_domains("http://www.dnsdb.org/f/#{CGI.escape(domain)}.dnsdb.org/", domain)
  end
  domains = deep_domains.sort.uniq
end

if domains.size.zero?
  puts "No domains found for #{ARGV.first}"
  exit
end

longest_domain_length = domains.max_by(&:length).length
verbose("Resolving domains with nameservers: #{@options[:nameservers].join(', ')}...")
printf("\e[1m%-#{longest_domain_length}s\t%-15s\tPorts\n\e[0m", "Domain", "IP")
resolver = Resolv::DNS.new(:nameserver => @options[:nameservers])
domains.each do |domain|
  begin
    if ip = resolver.getaddress("#{domain}.")
      open_ports = []
      if !@options[:ports].empty?
        @options[:ports].each do |port|
          open_ports << port if port_open?(ip.to_s, port)
        end
        next if open_ports.empty?
      end
      printf("%-#{longest_domain_length}s\t%-15s\t%s\n", domain, ip, open_ports.join(", "))
    end
  rescue Resolv::ResolvError
  end
end