scrape kanopy

scrape-kanopy.js

var results = [];
for (let pg =1;pg < 50_000; pg++) {
  var res = await fetch(`https://www.kanopy.com/kapi/search/videos?sort=popular&rfp=exclude&domainId=2290&isKids=false&page=${pg}&perPage=40`, {
      "credentials": "include",
      "headers": {
          "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:137.0) Gecko/20100101 Firefox/137.0",
          "Accept": "application/json, text/plain, */*",
          "Accept-Language": "en-US,en;q=0.5",
          "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjp7InVpZCI6IjMzMDU3NjkiLCJpZGVudGl0eV9pZCI6IjE3OTMxMDI2OCIsInZpc2l0b3JfaWQiOiIxNzQzOTExNDcwNTk5MDE5MjI1Iiwic2Vzc2lvbl9pZCI6IjE3NDUxODkxOTI0MjQwNTE3NzIiLCJjb25uZWN0aW9uX2lkIjoiMTc0NTE4OTE5MjQyNDA1MTc3MiIsImt1aV91c2VyIjoxLCJyb2xlcyI6WyJjb21Vc2VyIl19LCJpYXQiOjE3NDUxODkxOTIsImV4cCI6MjA2MDU0OTE5MiwiaXNzIjoia2FwaSJ9.dS0SQ32ZF5Vezr2Ha3GCvgFZ5tbzraAjIvRm_5LFCTI",
          "X-Version": "web/prod/5.3.0/2025-03-24-13-06-01",
          "Sec-Fetch-Dest": "empty",
          "Sec-Fetch-Mode": "cors",
          "Sec-Fetch-Site": "same-origin"
      },
      "referrer": "https://www.kanopy.com/en/sfpl/search",
      "method": "GET",
      "mode": "cors"
  });
  var items = (await res.json()).list
  if (items.length == 0) {
    break
  }
  results.push(...items)
  await new Promise(r => setTimeout(r, 5_000))
}
var fs = require("node:fs/promises")
await fs.writeFile("kanopy.json", JSON.stringify(results,null,2))