verifyBot.go

package main

import (
	"bytes"
	"fmt"
	"net"
	"strings"
)

func main() {
	fmt.Println(verifyBot("66.249.64.118", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"))
	fmt.Println(verifyBot("66.249.89.180", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)"))
	fmt.Println(verifyBot("66.249.92.71", "Mediapartners-Google"))
	fmt.Println(verifyBot("13.66.139.1", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"))
	fmt.Println(verifyBot("91.99.75.10", "Mediapartners-Google")) // false
	fmt.Println(verifyBot("66.249.92.71", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1")) // false
}

// verifyBot used to check userAgent for valid bot values and verify IP address
// with a reverse and forward DNS lookup
func verifyBot(ip, userAgent string) bool {
	// check userAgent for bot values
	if checkUserAgent(userAgent) {

		// reverse dns lookup
		rev, err := net.LookupAddr(ip)
		if err != nil {
			panic(err)
		}

		// check host name to be valid bot host name
		if checkHostName(rev[0]) {
			// forward dns lookup
			frw, err := net.LookupIP(rev[0])
			if err != nil {
				panic(err)
			}
			// check forward dns result with the original bot address
			parsedIP := net.ParseIP(ip)
			for _, res := range frw {
				if bytes.Compare(res, parsedIP) == 0 {
					// TODO add IP address to the whitelist
					return true
				}
			}
		}
	}
	return false
}

// checkUserAgent is used to check bot string patterns in userAgent
func checkUserAgent(userAgent string) bool {
	matchingWords := []string{"bot", "Google"}
	for _, word := range matchingWords {
		if strings.Contains(userAgent, word) {
			return true
		}
	}
	return false

}

// checkHostName for valid bot hostname for google and Bingbot
func checkHostName(host string) bool {
	matchingWords := []string{"msn.com.", "google.com.", "googlebot.com."}
	for _, word := range matchingWords {
		if strings.HasSuffix(host, word) {
			return true
		}
	}
	return false
}