Created
November 11, 2016 14:30
-
-
Save marshyon/1900ca3f9f374991a12d1af9285f7ab8 to your computer and use it in GitHub Desktop.
golang concurrent http client with worker and generator 'load balance' mechanism and pool of workers
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"fmt" | |
"io/ioutil" | |
"net/http" | |
) | |
func getPage(url string) (int, error) { | |
resp, err := http.Get(url) | |
if err != nil { | |
return 0, err | |
} | |
defer resp.Body.Close() | |
body, err := ioutil.ReadAll(resp.Body) | |
if err != nil { | |
return 0, err | |
} | |
return len(body), nil | |
} | |
func worker(urlCh chan string, sizeCh chan string, id int) { | |
for { | |
url := <-urlCh | |
length, err := getPage(url) | |
if err == nil { | |
sizeCh <- fmt.Sprintf("[%d] %s has length %d ", id, url, length) | |
} else { | |
sizeCh <- fmt.Sprintf("Error getting %s :: %s", url, err) | |
} | |
} | |
} | |
func generator(url string, urlCh chan string) { | |
urlCh <- url | |
} | |
func main() { | |
urlCh := make(chan string) | |
sizeCh := make(chan string) | |
urls := []string{"https://duckduckgo.com", "https://www.google.com", | |
"https://www.yahoo.com", "http://www.bing.com", "http://www.bbc.co.uk", | |
"http://www.yell.com", "https://golang.org", "http://golangweekly.com/", | |
"http://marcio.io", "http://blog.davidvassallo.me", | |
"https://www.safaribooksonline.com", "https://dzone.com", | |
"http://www.try.backtrace.io", "http://youtube.com", | |
"https://www.cockroachlabs.com"} | |
for i := 0; i < 10; i++ { | |
go worker(urlCh, sizeCh, i) | |
} | |
for _, url := range urls { | |
go generator(url, urlCh) | |
} | |
for i := 0; i < len(urls); i++ { | |
fmt.Printf("%s\n", <-sizeCh) | |
} | |
fmt.Printf("done [%d] urls\n", len(urls)) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment