gatsby-node.js

/**
 * Implement Gatsby's Node APIs in this file.
 *
 * See: https://www.gatsbyjs.org/docs/node-apis/
 */

// You can delete this file if you're not using it

const fetch = require("node-fetch");
const axios = require("axios");

function getRateLimitedAxiosInstance() {
    const rateLimit = 0;
    let lastCalled = undefined;
  
    const rateLimiter = (call) => {
      // Thank you to https://stackoverflow.com/questions/43482639/throttling-axios-requests
      const now = Date.now();
      if (lastCalled) {
        lastCalled += rateLimit;
        const wait = (lastCalled - now);
        if (wait > 0) {
          return new Promise((resolve) => setTimeout(() => resolve(call), wait));
        }
      }
      lastCalled = now;
      return call;
    }
  
    const api = axios.create({
      baseURL: `http://playbattleroyale.test/api`,
    });
  
    api.interceptors.request.use(rateLimiter);
  
    return api;
}

exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }) => {
    const { createNode } = actions
  
    const api = getRateLimitedAxiosInstance()

    const initialResponse = await api.get(`/clips/total`)
    const totalClips = initialResponse.data.total
    const numPages = Math.ceil(totalClips / 10000);
    console.log(numPages)

    let clipsProcessed = 0
    let currentPage = 1
    let nodes = []

    try {

        while (currentPage <= numPages) {
            
            const response = await api.get(`/clips/all?page=${currentPage}`)
            const data = response.data

            if (data.data.length < 1) break

            data.data.forEach(clip => {
                const nodeMeta = {
                    id: createNodeId(`pbr-clip-${clip.id}`),
                    parent: null,
                    children: [],
                    internal: {
                        type: `Clip`,
                        mediaType: `text/html`,
                        content: JSON.stringify(clip),
                        contentDigest: createContentDigest(clip)
                    }
                }
                console.log(`${clip.twitch_clip_id} ${clip.id}`)
                const node = Object.assign({}, clip, nodeMeta)
                nodes.push(node)
                clipsProcessed++
            });

            currentPage++
            console.log(currentPage)
        }

        console.log(nodes.length)
        nodes.forEach(node => {
            createNode(node)
            console.log(`Created node for clip ${node.id}`)
        })
    } catch (error) {
        console.error(error)
        process.exit(1)
    }
}