Last active
January 17, 2023 02:58
-
-
Save MaximRouiller/b5613327a28ae9e052c0425f636ffc34 to your computer and use it in GitHub Desktop.
Automatically download files to Azure Blob Storage using Copy From URL API with C#
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using Azure.Storage.Blobs; | |
using System; | |
using System.IO; | |
using System.Threading.Tasks; | |
namespace ConsoleApp2 | |
{ | |
class Program | |
{ | |
// Required .NET Package: Azure.Storage.Blobs version >=12 | |
// API Docs: https://docs.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url | |
static async Task Main(string[] args) | |
{ | |
string connectionString = Environment.GetEnvironmentVariable("AZURE_CONNECTION_STRING", EnvironmentVariableTarget.Process); | |
var filesToProcess = new[] { | |
"http://deepyeti.ucsd.edu/jianmo/amazon/categoryFilesSmall/Grocery_and_Gourmet_Food.csv", // ~200MB | |
"http://dmery.sitios.ing.uc.cl/images/GDXray/Castings.zip", // ~295MB | |
"http://dmery.sitios.ing.uc.cl/images/GDXray/Baggages.zip" // ~3GB | |
}; | |
Console.WriteLine("Copying Blob from URL (size limited to 250MB (?) )"); | |
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString); | |
string containerName = $"data-{DateTime.UtcNow:yyyyMMddHHmm}"; | |
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName); | |
await containerClient.CreateIfNotExistsAsync(); | |
foreach (var fileToProcess in filesToProcess) | |
{ | |
string fileName = Path.GetFileName(fileToProcess); | |
var blobClient = containerClient.GetBlobClient(fileName); | |
blobClient.StartCopyFromUri(new Uri(fileToProcess)); | |
} | |
Console.WriteLine($"Operation started. Closing application."); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment