Created
October 31, 2022 05:40
-
-
Save shuson/afea9560ad6c895dcac2b3d809923fd1 to your computer and use it in GitHub Desktop.
S3 download file more than 2GB using nodejs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require("fs") | |
const s3 = require("aws-s3") // blabla | |
const totalSize = await getFileSize(file) // if this is more than 2147483638 | |
logger.info("downloading huge file ... " + file) | |
const partSize = 547483640 | |
const saveFilePath = "path_to_save.file" | |
const parts = Math.ceil(totalSize/partSize) | |
const writeStream = fs.createWriteStream(saveFilePath, {flags:'a'}) | |
for(let i = 0; i < (parts-1); i++) { | |
const endBytes = (i+1)*partSize > totalSize? totalSize : (i+1)*partSize | |
params['Range'] = `bytes=${i*partSize}-${endBytes}` | |
const { | |
Body | |
} = await s3.getObject(params).promise() | |
writeStream.write(Body) | |
} | |
writeStream.end() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment