Sample Go Script to Check for Domain Frontable Domains

checkDF.go

package main

import (
    //"sync"
    "fmt"
    "net/http"
    "strings"
    "bufio"
    "io/ioutil"
    "archive/zip"
    "path/filepath"
    "io"
    "os"
    //"net/url"
    "github.com/remeh/sizedwaitgroup"
    "bytes"
    "encoding/json"
    "log"
    "text/tabwriter"
    )

func testURL(domainName string) string{
	var url = "https://sitereview.bluecoat.com/resource/lookup"
	domainName = strings.TrimSpace(domainName)		

	type Messsage struct {
		URL               string      `json:"url"`
		Unrated           bool        `json:"unrated"`
		CurTrackingID     int         `json:"curTrackingId"`
		Locked            bool        `json:"locked"`
		LockedMessage     interface{} `json:"lockedMessage"`
		LockedSpecialNote interface{} `json:"lockedSpecialNote"`
		Multiple          bool        `json:"multiple"`
		MultipleMessage   interface{} `json:"multipleMessage"`
		RateDate          string      `json:"rateDate"`
		Categorization    []struct {
			Num  int    `json:"num"`
			Name string `json:"name"`
		} `json:"categorization"`
		ThreatriskLevel     string      `json:"threatriskLevel"`
		ThreatriskLevelEn   string      `json:"threatriskLevelEn"`
		Linkable            bool        `json:"linkable"`
		SecurityCategory    bool        `json:"securityCategory"`
		SecurityCategoryIds []int       `json:"securityCategoryIds"`
		Suggestion          interface{} `json:"suggestion"`
		ResolvedDetail      struct {
			ResolveEnabled bool   `json:"resolveEnabled"`
			IPAddress      string `json:"ipAddress"`
		} `json:"resolvedDetail"`
		FollowedURL          interface{} `json:"followedUrl"`
		TranslatedCategories struct {
			De []struct {
				Num  int    `json:"num"`
				Name string `json:"name"`
			} `json:"de"`
			ZhTW []struct {
				Num  int    `json:"num"`
				Name string `json:"name"`
			} `json:"zh_TW"`
			Ja []struct {
				Num  int    `json:"num"`
				Name string `json:"name"`
			} `json:"ja"`
			En []struct {
				Num  int    `json:"num"`
				Name string `json:"name"`
			} `json:"en"`
			Fr []struct {
				Num  int    `json:"num"`
				Name string `json:"name"`
			} `json:"fr"`
			Es []struct {
				Num  int    `json:"num"`
				Name string `json:"name"`
			} `json:"es"`
			Zh []struct {
				Num  int    `json:"num"`
				Name string `json:"name"`
			} `json:"zh"`
		} `json:"translatedCategories"`
		TranslatedRateDates []struct {
			Locale string `json:"locale"`
			Text   string `json:"text"`
		} `json:"translatedRateDates"`
	}	
	
	values := map[string]string{"url": domainName, "captcha": ""}
	jsonValue, _ := json.Marshal(values)
	resp, err := http.Post(url, "application/json", bytes.NewBuffer(jsonValue))
	_ = err
    body, _ := ioutil.ReadAll(resp.Body)

	//https://mholt.github.io/json-to-go/
	var m Messsage

	if !strings.Contains(string(body),"errorMessage") {	
		if err := json.Unmarshal([]byte(string(body)), &m); err != nil {
			log.Fatal(err)
		}	

		var categoryList []string
		for _, v := range m.Categorization {
			categoryList = append(categoryList,v.Name)
		}
		var catStr = strings.Join(categoryList[:],",")
		return catStr
	} else {
		return " Captcha error"
	}
}

func readLines(path string) ([]string, error) {
  file, err := os.Open(path)
  if err != nil {
    return nil, err
  }
  defer file.Close()

  var lines []string
  scanner := bufio.NewScanner(file)
  for scanner.Scan() {
    lines = append(lines, scanner.Text())
  }
  return lines, scanner.Err()
}

func unzip(archive, target string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}
	for _, file := range reader.File {
		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, file.Mode())
			continue
		}
		fileReader, err := file.Open()
		if err != nil {
			return err
		}
		defer fileReader.Close()
		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			return err
		}
		defer targetFile.Close()
		if _, err := io.Copy(targetFile, fileReader); err != nil {
			return err
		}
	}
	return nil
}

func downloadFromUrl(url string) {
	tokens := strings.Split(url, "/")
	fileName := tokens[len(tokens)-1]
	fmt.Println("Downloading", url, "to", fileName)

	// TODO: check file existence first with io.IsExist
	output, err := os.Create(fileName)
	if err != nil {
		fmt.Println("Error while creating", fileName, "-", err)
		return
	}
	defer output.Close()
	response, err := http.Get(url)
	if err != nil {
		fmt.Println("Error while downloading", url, "-", err)
		return
	}
	defer response.Body.Close()
	n, err := io.Copy(output, response.Body)
	if err != nil {
		fmt.Println("Error while downloading", url, "-", err)
		return
	}
	fmt.Println(n, "bytes downloaded.")
}
func checkDomain(domainName string) (string) {   
	var url = "https://"+domainName+"/images/I/01rgQ3jqo7L.css"
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err==nil {
		req.Header.Add("Host", "images-na.ssl-images-amazon.com")
		req.Host = "images-na.ssl-images-amazon.com"
		req.Header.Add("Connection", "close")
		resp, err1 := client.Do(req)
		//defer resp.Body.Close()
		if err1==nil {
			if resp.StatusCode==200 {
				body, err2 := ioutil.ReadAll(resp.Body)
				if string(body)=="#octane-aui-example-id{color:red}" {
					//fmt.Println(domainName)
					//x =testURL(domainName)
					return domainName
				}
				_ = err2
			}
		}
		_ = err
	}
	return ""
}
func main() {
	var domainList []string
	var downloadUrl="http://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
	tokens := strings.Split(downloadUrl, "/")
	fileName := tokens[len(tokens)-1]
	if _, err := os.Stat("top-1m.csv.zip"); os.IsNotExist(err) {
		fmt.Println("[*] Downloading", downloadUrl, "to", fileName)
		downloadFromUrl(downloadUrl)
	}
	pwd, err := os.Getwd()
	if err == nil {
		if _, err := os.Stat("top-1m.csv"); os.IsNotExist(err) {
			fmt.Println("[*] Uncompressing ",fileName)
			unzip(pwd+"/"+fileName, pwd)
		}
	}
	var pFilename = "top-1m.csv"
	lines, err2 := readLines(pFilename)
	for _, v := range lines {
		v=strings.TrimSpace(v)
		if len(v)>0 {
			var s1=strings.Split(v,",")
			domainList = append(domainList, s1[1])
		}
	}		
	_ = err2

	var maxConcurrency=100
	swg := sizedwaitgroup.New(maxConcurrency)

	//var wg sync.WaitGroup
	//wg.Add(len(domainList))

	const padding = 3
	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.Debug)

	for i := 0; i < len(domainList); i++ {
		swg.Add()
		go func(i int) {
		       defer swg.Done()
        	   //defer wg.Done()
 		       val := domainList[i]
		       //fmt.Println("Checking: "+val)
		       var z = checkDomain(val) 
		       if len(z)>0 {
			       var g = testURL(z)
				   fmt.Fprintln(w, val+"\t"+g)
				   _= g
			       w.Flush()
				} 			
		 }(i)
	}
	//wg.Wait()
	swg.Wait()
}