Instantly share code, notes, and snippets.
Created
September 12, 2013 13:54
-
Star
0
(0)
You must be signed in to star a gist -
Fork
1
(1)
You must be signed in to fork a gist
-
Save shivkumarganesh/6537779 to your computer and use it in GitHub Desktop.
This Program helps you to download all the StackExchangeTags from the StackExchange Website. Kinda jumbled up to look but works well. You just need to get the API key and add it with a valid MongoDB url.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
npm install -g superagent && | |
npm install -g mongoose && | |
echo Please include these in your Project directory so as to be referred by StackExchangeTags.js |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var siteList={ | |
"items": | |
[ { "api_site_parameter": "stackoverflow" }, | |
{ "api_site_parameter": "serverfault" }, | |
{ "api_site_parameter": "superuser" }, | |
{ "api_site_parameter": "meta.stackoverflow" }, | |
{ "api_site_parameter": "webapps" }, | |
{ "api_site_parameter": "meta.webapps" }, | |
{ "api_site_parameter": "gaming" }, | |
{ "api_site_parameter": "meta.gaming" }, | |
{ "api_site_parameter": "webmasters" }, | |
{ "api_site_parameter": "meta.webmasters" }, | |
{ "api_site_parameter": "cooking" }, | |
{ "api_site_parameter": "meta.cooking" }, | |
{ "api_site_parameter": "gamedev" }, | |
{ "api_site_parameter": "meta.gamedev" }, | |
{ "api_site_parameter": "photo" }, | |
{ "api_site_parameter": "meta.photo" }, | |
{ "api_site_parameter": "stats" }, | |
{ "api_site_parameter": "meta.stats" }, | |
{ "api_site_parameter": "math" }, | |
{ "api_site_parameter": "meta.math" }, | |
{ "api_site_parameter": "diy" }, | |
{ "api_site_parameter": "meta.diy" }, | |
{ "api_site_parameter": "meta.superuser" }, | |
{ "api_site_parameter": "meta.serverfault" }, | |
{ "api_site_parameter": "gis" }, | |
{ "api_site_parameter": "meta.gis" }, | |
{ "api_site_parameter": "tex" }, | |
{ "api_site_parameter": "meta.tex" }, | |
{ "api_site_parameter": "askubuntu" }, | |
{ "api_site_parameter": "meta.askubuntu" }, | |
{ "api_site_parameter": "money" }, | |
{ "api_site_parameter": "meta.money" }, | |
{ "api_site_parameter": "english" }, | |
{ "api_site_parameter": "meta.english" }, | |
{ "api_site_parameter": "stackapps" }, | |
{ "api_site_parameter": "ux" }, | |
{ "api_site_parameter": "meta.ux" }, | |
{ "api_site_parameter": "unix" }, | |
{ "api_site_parameter": "meta.unix" }, | |
{ "api_site_parameter": "wordpress" }, | |
{ "api_site_parameter": "meta.wordpress" }, | |
{ "api_site_parameter": "cstheory" }, | |
{ "api_site_parameter": "meta.cstheory" }, | |
{ "api_site_parameter": "apple" }, | |
{ "api_site_parameter": "meta.apple" }, | |
{ "api_site_parameter": "rpg" }, | |
{ "api_site_parameter": "meta.rpg" }, | |
{ "api_site_parameter": "bicycles" }, | |
{ "api_site_parameter": "meta.bicycles" }, | |
{ "api_site_parameter": "programmers" }, | |
{ "api_site_parameter": "meta.programmers" }, | |
{ "api_site_parameter": "electronics" }, | |
{ "api_site_parameter": "meta.electronics" }, | |
{ "api_site_parameter": "android" }, | |
{ "api_site_parameter": "meta.android" }, | |
{ "api_site_parameter": "answers.onstartups" }, | |
{ "api_site_parameter": "meta.answers.onstartups" }, | |
{ "api_site_parameter": "boardgames" }, | |
{ "api_site_parameter": "meta.boardgames" }, | |
{ "api_site_parameter": "physics" }, | |
{ "api_site_parameter": "meta.physics" }, | |
{ "api_site_parameter": "homebrew" }, | |
{ "api_site_parameter": "meta.homebrew" }, | |
{ "api_site_parameter": "security" }, | |
{ "api_site_parameter": "meta.security" }, | |
{ "api_site_parameter": "writers" }, | |
{ "api_site_parameter": "meta.writers" }, | |
{ "api_site_parameter": "avp" }, | |
{ "api_site_parameter": "meta.avp" }, | |
{ "api_site_parameter": "graphicdesign" }, | |
{ "api_site_parameter": "meta.graphicdesign" }, | |
{ "api_site_parameter": "dba" }, | |
{ "api_site_parameter": "meta.dba" }, | |
{ "api_site_parameter": "scifi" }, | |
{ "api_site_parameter": "meta.scifi" }, | |
{ "api_site_parameter": "codereview" }, | |
{ "api_site_parameter": "meta.codereview" }, | |
{ "api_site_parameter": "codegolf" }, | |
{ "api_site_parameter": "meta.codegolf" }, | |
{ "api_site_parameter": "quant" }, | |
{ "api_site_parameter": "meta.quant" }, | |
{ "api_site_parameter": "pm" }, | |
{ "api_site_parameter": "meta.pm" }, | |
{ "api_site_parameter": "skeptics" }, | |
{ "api_site_parameter": "meta.skeptics" }, | |
{ "api_site_parameter": "fitness" }, | |
{ "api_site_parameter": "meta.fitness" }, | |
{ "api_site_parameter": "drupal" }, | |
{ "api_site_parameter": "meta.drupal" }, | |
{ "api_site_parameter": "mechanics" }, | |
{ "api_site_parameter": "meta.mechanics" }, | |
{ "api_site_parameter": "parenting" }, | |
{ "api_site_parameter": "meta.parenting" }, | |
{ "api_site_parameter": "sharepoint" }, | |
{ "api_site_parameter": "meta.sharepoint" }, | |
{ "api_site_parameter": "music" }, | |
{ "api_site_parameter": "meta.music" }, | |
{ "api_site_parameter": "sqa" }, | |
{ "api_site_parameter": "meta.sqa" }, | |
{ "api_site_parameter": "judaism" } , | |
{ "api_site_parameter": "meta.freelancing" }, | |
{ "api_site_parameter": "blender" }, | |
{ "api_site_parameter": "meta.blender" }, | |
{ "api_site_parameter": "mathoverflow.net" }, | |
{ "api_site_parameter": "meta.mathoverflow.net" }, | |
{ "api_site_parameter": "space" }, | |
{ "api_site_parameter": "meta.space" }, | |
{ "api_site_parameter": "digitalfabrication" }, | |
{ "api_site_parameter": "meta.digitalfabrication" }, | |
{ "api_site_parameter": "sound" }, | |
{ "api_site_parameter": "meta.sound" }, | |
{ "api_site_parameter": "meta.judaism" }, | |
{ "api_site_parameter": "german" }, | |
{ "api_site_parameter": "meta.german" }, | |
{ "api_site_parameter": "japanese" }, | |
{ "api_site_parameter": "meta.japanese" }, | |
{ "api_site_parameter": "philosophy" }, | |
{ "api_site_parameter": "meta.philosophy" }, | |
{ "api_site_parameter": "gardening" }, | |
{ "api_site_parameter": "meta.gardening" }, | |
{ "api_site_parameter": "travel" }, | |
{ "api_site_parameter": "meta.travel" }, | |
{ "api_site_parameter": "productivity" }, | |
{ "api_site_parameter": "meta.productivity" }, | |
{ "api_site_parameter": "crypto" }, | |
{ "api_site_parameter": "meta.crypto" }, | |
{ "api_site_parameter": "dsp" }, | |
{ "api_site_parameter": "meta.dsp" }, | |
{ "api_site_parameter": "french" }, | |
{ "api_site_parameter": "meta.french" }, | |
{ "api_site_parameter": "christianity" }, | |
{ "api_site_parameter": "meta.christianity" }, | |
{ "api_site_parameter": "bitcoin" }, | |
{ "api_site_parameter": "meta.bitcoin" }, | |
{ "api_site_parameter": "linguistics" }, | |
{ "api_site_parameter": "meta.linguistics" }, | |
{ "api_site_parameter": "hermeneutics" }, | |
{ "api_site_parameter": "meta.hermeneutics" }, | |
{ "api_site_parameter": "history" }, | |
{ "api_site_parameter": "meta.history" }, | |
{ "api_site_parameter": "bricks" }, | |
{ "api_site_parameter": "meta.bricks" }, | |
{ "api_site_parameter": "spanish" }, | |
{ "api_site_parameter": "meta.spanish" }, | |
{ "api_site_parameter": "scicomp" }, | |
{ "api_site_parameter": "meta.scicomp" }, | |
{ "api_site_parameter": "movies" }, | |
{ "api_site_parameter": "meta.movies" }, | |
{ "api_site_parameter": "chinese" }, | |
{ "api_site_parameter": "meta.chinese" }, | |
{ "api_site_parameter": "biology" }, | |
{ "api_site_parameter": "meta.biology" }, | |
{ "api_site_parameter": "poker" }, | |
{ "api_site_parameter": "meta.poker" }, | |
{ "api_site_parameter": "mathematica" }, | |
{ "api_site_parameter": "meta.mathematica" }, | |
{ "api_site_parameter": "cogsci" }, | |
{ "api_site_parameter": "meta.cogsci" }, | |
{ "api_site_parameter": "outdoors" }, | |
{ "api_site_parameter": "meta.outdoors" }, | |
{ "api_site_parameter": "martialarts" }, | |
{ "api_site_parameter": "meta.martialarts" }, | |
{ "api_site_parameter": "sports" }, | |
{ "api_site_parameter": "meta.sports" }, | |
{ "api_site_parameter": "academia" }, | |
{ "api_site_parameter": "meta.academia" }, | |
{ "api_site_parameter": "cs" }, | |
{ "api_site_parameter": "meta.cs" }, | |
{ "api_site_parameter": "workplace" }, | |
{ "api_site_parameter": "meta.workplace" }, | |
{ "api_site_parameter": "windowsphone" }, | |
{ "api_site_parameter": "meta.windowsphone" }, | |
{ "api_site_parameter": "chemistry" }, | |
{ "api_site_parameter": "meta.chemistry" }, | |
{ "api_site_parameter": "chess" }, | |
{ "api_site_parameter": "meta.chess" }, | |
{ "api_site_parameter": "raspberrypi" }, | |
{ "api_site_parameter": "meta.raspberrypi" }, | |
{ "api_site_parameter": "russian" }, | |
{ "api_site_parameter": "meta.russian" }, | |
{ "api_site_parameter": "islam" }, | |
{ "api_site_parameter": "meta.islam" }, | |
{ "api_site_parameter": "salesforce" }, | |
{ "api_site_parameter": "meta.salesforce" }, | |
{ "api_site_parameter": "patents" }, | |
{ "api_site_parameter": "meta.patents" }, | |
{ "api_site_parameter": "genealogy" }, | |
{ "api_site_parameter": "meta.genealogy" }, | |
{ "api_site_parameter": "robotics" }, | |
{ "api_site_parameter": "meta.robotics" }, | |
{ "api_site_parameter": "expressionengine" }, | |
{ "api_site_parameter": "meta.expressionengine" }, | |
{ "api_site_parameter": "politics" }, | |
{ "api_site_parameter": "meta.politics" }, | |
{ "api_site_parameter": "anime" }, | |
{ "api_site_parameter": "meta.anime" }, | |
{ "api_site_parameter": "magento" }, | |
{ "api_site_parameter": "meta.magento" }, | |
{ "api_site_parameter": "ell" }, | |
{ "api_site_parameter": "meta.ell" }, | |
{ "api_site_parameter": "sustainability" }, | |
{ "api_site_parameter": "meta.sustainability" }, | |
{ "api_site_parameter": "tridion" }, | |
{ "api_site_parameter": "meta.tridion" }, | |
{ "api_site_parameter": "reverseengineering" }, | |
{ "api_site_parameter": "meta.reverseengineering" }, | |
{ "api_site_parameter": "networkengineering" }, | |
{ "api_site_parameter": "meta.networkengineering" }, | |
{ "api_site_parameter": "opendata" }, | |
{ "api_site_parameter": "meta.opendata" }, | |
{ "api_site_parameter": "freelancing" } ]} | |
console.log(siteList.items.length); | |
var request=require('./superagent'); | |
var mongoose = require('./mongoose'); | |
mongoose.connect('mongodb://localhost/StackExchangeTags'); | |
var db = mongoose.connection; | |
db.on('error', console.error.bind(console, 'connection error:')); | |
db.once('open', function callback () { | |
console.log('Connected'); | |
}); | |
//Schema for Tags | |
var tagSchema = mongoose.Schema({ | |
name:String, | |
count:Number, | |
is_required:Boolean, | |
is_moderator_only:Boolean, | |
has_synonyms:Boolean}); | |
var saveSchema = function(Model){ | |
Model.save(); | |
}; | |
var time=0; | |
var count=0; | |
var i=1; | |
var req=function(){ | |
var Tags = mongoose.model(siteList.items[count].api_site_parameter, tagSchema); | |
request.get('http://api.stackexchange.com/2.1/tags?page='+i+'&pagesize=100&fromdate=1199145600&todate=1377648000&order=asc&sort=popular&site='+siteList.items[count].api_site_parameter+'&key=VA6c0LmO6lmPdYLk3KLp6A((') | |
.set('Accept', 'application/json') | |
.end(function(res){ | |
if (res.ok && res.body.items.length!==0) { | |
console.log('yay got ' + res.body); | |
var jsonResponse = res.body; | |
console.log(jsonResponse.items.length); | |
for(var j=0;j<jsonResponse.items.length;j++) | |
{ | |
var model = new Tags(jsonResponse.items[j]); | |
saveSchema(model); | |
} | |
i++; | |
console.log("Number of Iteration ---- > "+i); | |
if(jsonResponse.backoff !== null && jsonResponse.backoff !== "" && jsonResponse.backoff !== undefined){ | |
time=time+jsonResponse.backoff; | |
setTimeout(function() { | |
console.log('BACKOFF IS SET'); | |
req(); | |
}, time); | |
} | |
else{ | |
setTimeout(function() { | |
console.log('BACKOFF IS NOT SET----- :)'); | |
req(); | |
}, time); | |
} | |
} else { | |
count++; | |
i=1; | |
console.log('One Site Complete'); | |
setTimeout(function() { | |
console.log('BACKOFF IS SET'); | |
req(); | |
}, 2000); | |
} | |
}); | |
} | |
req(); | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment