Skip to content

Instantly share code, notes, and snippets.

@marshyon
Created November 11, 2016 14:30
Show Gist options
  • Save marshyon/1900ca3f9f374991a12d1af9285f7ab8 to your computer and use it in GitHub Desktop.
Save marshyon/1900ca3f9f374991a12d1af9285f7ab8 to your computer and use it in GitHub Desktop.
golang concurrent http client with worker and generator 'load balance' mechanism and pool of workers
package main
import (
"fmt"
"io/ioutil"
"net/http"
)
func getPage(url string) (int, error) {
resp, err := http.Get(url)
if err != nil {
return 0, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return 0, err
}
return len(body), nil
}
func worker(urlCh chan string, sizeCh chan string, id int) {
for {
url := <-urlCh
length, err := getPage(url)
if err == nil {
sizeCh <- fmt.Sprintf("[%d] %s has length %d ", id, url, length)
} else {
sizeCh <- fmt.Sprintf("Error getting %s :: %s", url, err)
}
}
}
func generator(url string, urlCh chan string) {
urlCh <- url
}
func main() {
urlCh := make(chan string)
sizeCh := make(chan string)
urls := []string{"https://duckduckgo.com", "https://www.google.com",
"https://www.yahoo.com", "http://www.bing.com", "http://www.bbc.co.uk",
"http://www.yell.com", "https://golang.org", "http://golangweekly.com/",
"http://marcio.io", "http://blog.davidvassallo.me",
"https://www.safaribooksonline.com", "https://dzone.com",
"http://www.try.backtrace.io", "http://youtube.com",
"https://www.cockroachlabs.com"}
for i := 0; i < 10; i++ {
go worker(urlCh, sizeCh, i)
}
for _, url := range urls {
go generator(url, urlCh)
}
for i := 0; i < len(urls); i++ {
fmt.Printf("%s\n", <-sizeCh)
}
fmt.Printf("done [%d] urls\n", len(urls))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment