Node.js script for querying GitHub Search API v3

github-api-search.js

// API docs: https://developer.github.com/v3/search/
// search docs: https://help.github.com/en/github/searching-for-information-on-github/searching-on-github

// you might have to limit the number of requests
// if trying to send too many at once

const request = require('request');
const _ = require('lodash');
const fs = require('fs');

const opts = {
    headers: {
        // user agent string can be something like 'Repository search'
        'User-Agent': '<USER_AGENT_STRING>'
        // optional: add your access token here
        //
        // providing an access token will allow you
        // to send more requests per minute:
        // https://developer.github.com/v3/search/#rate-limit
        //
        // Authorization: `token <ACCESS_TOKEN_HERE>`
    }
};

// example url, adjust to your needs
// it's always a good idea to include per_page=100 (maximum allowed)
// to minimise number of requests
const initUrl = 'https://api.github.com/search/repositories?q=user:ilokhov&per_page=100';

sendRequest({ ...opts, ...{ url: initUrl } }, []);

function sendRequest(opts, results) {
    console.log(`request: ${opts.url}`);

    request(opts, (err, response, body) => {
        if (err) throw err;

        // add to results from current request
        results.push(...getContent(body));

        // check if we have more pages available
        const responseHeader = response.headers;
        const link = responseHeader.link;

        // there are no addition pages
        if (!link) {
            writeOutput(results);
        } else {
            // check if we have next page
            const nextRelIndex = link.indexOf('rel="next"');

            // no next page
            if (nextRelIndex === -1) {
                writeOutput(results);
            } else {
                // get next page url
                const nextStartIndex = link.lastIndexOf('<', nextRelIndex);
                const nextEndIndex = link.lastIndexOf('>', nextRelIndex);
                const nextUrl = link.substring(
                    nextStartIndex + 1,
                    nextEndIndex
                );

                // send next request
                sendRequest({ ...opts, ...{ url: nextUrl } }, results);
            }
        }
    });
}

function getContent(body) {
    const items = JSON.parse(body).items;

    // example value to pick, adjust to your needs
    return items.map(item => _.pick(item, ['created_at']));
}

function writeOutput(results) {
    console.log(results);
    fs.writeFile('output.json', JSON.stringify(results), err => {
        if (err) throw err;
        console.log('file written!');
    });
}