Improvements to source made available on pythonprogramming.net tutorial series. Original source can be found here: https://pythonprogramming.net/go/concurrency-web-app-go-language-programming-tutorial/
Last active
July 12, 2018 08:04
-
-
Save tpsrep0rts/5bcc9eb8e621fccfa8cec3d12c674ddd to your computer and use it in GitHub Desktop.
Optimized news aggregator
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <head> | |
| <script type="text/javascript" charset="utf8" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script> | |
| <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.16/css/jquery.dataTables.css"> | |
| <script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.16/js/jquery.dataTables.js"></script> | |
| </head> | |
| <h1>{{.Title}}</h1> | |
| <table id="example" class="display" cellspacing="0" width="100%"> | |
| <col width="35%"> | |
| <col width="65%"> | |
| <thead> | |
| <tr> | |
| <th>Title</th> | |
| <th>Keywords</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| {{ range $key, $value := .News }} | |
| <tr> | |
| <td><a href="{{ $value.Location }}" target='_blank'>{{ $key }}</td> | |
| <td>{{ $value.Keyword }}</td> | |
| </tr> | |
| {{ end }} | |
| </tbody> | |
| </table> | |
| <script>$(document).ready(function() { | |
| $('#example').DataTable(); | |
| } );</script> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "fmt" | |
| "net/http" | |
| "html/template" | |
| "newsfeed" | |
| ) | |
| func newsAggHandler(w http.ResponseWriter, r *http.Request) { | |
| p := newsfeed.NewsAggPage{Title: "Amazing News Aggregator", News: newsfeed.GetNews()} | |
| t, _ := template.ParseFiles("aggregatorfinish.html") | |
| t.Execute(w, p) | |
| } | |
| func indexHandler(w http.ResponseWriter, r *http.Request) { | |
| fmt.Fprintf(w, "<h1>Whoa, Go is neat!</h1>") | |
| } | |
| func main() { | |
| go newsfeed.RefreshNews() | |
| http.HandleFunc("/", indexHandler) | |
| http.HandleFunc("/agg/", newsAggHandler) | |
| http.ListenAndServe(":8000", nil) | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package newsfeed | |
| import ( | |
| "net/http" | |
| "encoding/xml" | |
| "io/ioutil" | |
| "sync" | |
| "time" | |
| ) | |
| var wg sync.WaitGroup | |
| type NewsMap struct { | |
| Keyword string | |
| Location string | |
| } | |
| type NewsAggPage struct { | |
| Title string | |
| News map[string]NewsMap | |
| } | |
| type Sitemapindex struct { | |
| Locations []string `xml:"sitemap>loc"` | |
| } | |
| type News struct { | |
| Titles []string `xml:"url>news>title"` | |
| Keywords []string `xml:"url>news>keywords"` | |
| Locations []string `xml:"url>loc"` | |
| } | |
| var news_map = make(map[string]NewsMap) | |
| func newsRoutine(c chan News, Location string){ | |
| defer wg.Done() | |
| var n News | |
| resp, _ := http.Get(Location) | |
| bytes, _ := ioutil.ReadAll(resp.Body) | |
| xml.Unmarshal(bytes, &n) | |
| resp.Body.Close() | |
| c <- n | |
| } | |
| func fetchNews() map[string]NewsMap { | |
| var s Sitemapindex | |
| resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml") | |
| bytes, _ := ioutil.ReadAll(resp.Body) | |
| xml.Unmarshal(bytes, &s) | |
| resp.Body.Close() | |
| queue := make(chan News, 30) | |
| for _, Location := range s.Locations { | |
| wg.Add(1) | |
| go newsRoutine(queue, Location) | |
| } | |
| wg.Wait() | |
| close(queue) | |
| for elem := range queue { | |
| for idx, _ := range elem.Keywords { | |
| news_map[elem.Titles[idx]] = NewsMap{elem.Keywords[idx], elem.Locations[idx]} | |
| } | |
| } | |
| return news_map | |
| } | |
| func RefreshNews() { | |
| for { | |
| news_map = fetchNews() | |
| time.Sleep(60 * time.Second) | |
| } | |
| } | |
| func GetNews() map[string]NewsMap { | |
| return news_map | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment