fetch all files from s3 bucket with given prefix and stream them to disk

fetch-s3-folder.js

// fetch-s3-folder.js
//
// usage: node fetch-s3-folder.js bucketName folderPath
//    ex: node fetcb-s3-folder.js our-bucket /my-folder/cool-subfolder
//
// to install dependencies, run following in same folder as this script:
// mkdir node_modules && npm install async knox mkdirp
// 
// also expects a `config.json` in same directory with AWS credentials

var Fs = require('fs');

var Async = require('async');
var Knox = require('knox');
var Mkdirp = require('mkdirp');

var bucketName = process.argv[2];
var prefix     = process.argv[3];

var config = require('./config.json');
config.bucket = bucketName;
var s3Client = Knox.createClient(config);

function log (message) {
  console.error('' + Date.now() + ' ' + message);
};

s3Client.list({ prefix: prefix }, function (err, data) {

  if (err) {
    throw err;
  }

  Mkdirp.sync(prefix);

  Async.mapLimit(data.Contents, 20, function (file, callback) {

    var outputStream = Fs.createWriteStream(file.Key);

    log('start ' + file.Key);

    s3Client.getFile(file.Key, function (err, res) {

      if (err) {
        return callback(err);
      }
      
      res.pipe(outputStream);
      
      res.on('end', function () {

        log('done ' + file.Key);

        return callback(null, file);
      });
    });
  },
  function (err, results) {

    if (err) {
      log('error');
      log(err.message);
      log(err);
      process.exit(1);
    }

    log('done');
  });
});