Last active
February 18, 2022 01:46
-
-
Save phil-kahrl/b3ddaee17b380356256f889432e44127 to your computer and use it in GitHub Desktop.
Recursive function for listing contents of an S3 bucket
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const AWS = require('aws-sdk'); | |
const s3 = new AWS.S3({ apiVersion: '2006-03-01', region: 'us-east-1' }); | |
/** | |
Recursive method to list all the objects in an S3 bucket which may be paginated. | |
**/ | |
const listObjects = ( | |
resultList, // This is the list of combined results from all calls. | |
NextContinuationToken // Tells S3 to get the next page of results. | |
) => { | |
console.log('list object called, NextContinuationToken = ' + NextContinuationToken) | |
// This function will return a Promise | |
return new Promise( (resolve, reject) => { | |
// initialize an empty result list on the first call | |
let list = resultList ? resultList : [] | |
const params = { | |
// a publicly accessible bucket for this example | |
Bucket: 'practical-recursion-example', | |
// default is MaxKeys is 1000 | |
// using 3 here to demonstrate use of recursion for multiple pages | |
MaxKeys: 3, | |
ContinuationToken: NextContinuationToken | |
}; | |
const request = s3.listObjectsV2(params); | |
request.send((err, data) => { | |
if (err) { | |
reject(err) | |
} else { | |
// combine the master list with the result of this function call. | |
list = list.concat(data.Contents) | |
if(data.IsTruncated) { | |
// if the data is truncated then call this function recursively | |
listObjects(list, data.NextContinuationToken).then( (list) => { | |
console.log( | |
'resolving for NextContinuationToken ' + | |
data.NextContinuationToken | |
) | |
resolve(list) | |
}) | |
} else { | |
/* | |
The termination condition for the recursion has been met, | |
(There are no more pages left in the bucket) so we resolve here. | |
*/ | |
console.log('done, resolving final result') | |
resolve(list) | |
} | |
} | |
}) | |
}) | |
} | |
module.exports = listObjects; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment