rust_vector_drain.rs

// For example, we have a vector of URLs we want to proccess it concurrently, but what if we have
// 10000 URLs, we do not want to spawn 10000 threads one time. Instead we limit it to 2-3
//       threads. Here's how I did it Rust looking at Golang code here: https://github.com/giovanni-orciuolo/cyberdrop-downloader/blob/484bf9fabb6544f4a1a7b640cef635bb37425faa/main.go#L268

const BATCH_SIZE: usize = 2; // How many URLs we want to process each step

fn main() {
    let mut albums = vec![
        "1".to_string(),
        "2".to_string(),
        "3".to_string(),
        "4".to_string(),
        "5".to_string(),
        "6".to_string(),
    ]; // Our URLs

    let mut idx: usize = 0; // we start always at 0

    // When 'albums' is empty we break
    while albums.len() != 0 {
        let queue: Vec<String>; // Create variable beforehand, because scopes!
        if BATCH_SIZE < albums.len() {
            queue = albums.drain(..BATCH_SIZE).collect(); // Drain vector from begining to BATCH_SIZE (0..2). Result is drained elements.
        } else {
            queue = albums.drain(..).collect();
        }
        println!("Queue: {:#?}", queue); // Our drained elements, do something with it, tokio::spawn etc
        println!("Albums: {:#?}", albums); // This is what we have left for next iteration
        idx += BATCH_SIZE; // Update index, next time we start
        println!("idx: {}", idx);
    }

    println!("Result: {:#?}", albums); // Should be empty vec []; We're done and proccessed every URL.
}