downloadSpiderDataset.fs

open System
open System.IO
open System.Diagnostics

let getImageLinks file =
    let text = File.ReadAllText file
    let links = text.Split "\n"
    links

let exec cmd = 
    // info for bash cmd to run
    let startInfo =
        ProcessStartInfo (
            FileName = "/bin/bash",
            Arguments = "-c \"" + cmd + "\"",
            UseShellExecute = false,
            RedirectStandardOutput = true
        )
    // run cmd
    let proc = new Process(StartInfo = startInfo)
    match proc.Start () with
    | true ->
        let result = proc.StandardOutput.ReadToEnd ()
        proc.WaitForExit ()
        (true, result)
    | false -> 
        (false, "terminal call went wrong!") 

let download link location =
    let cmd =  "cd " + location + "; wget " + link
    let _, msg = exec cmd
    ()

let run () =
    let links = getImageLinks "data/spiders.txt"
    let location = "/Users/btl787/00-local/07-datasets/micro-robot-imagenet/spiders"
    links |> Array.iter (fun link -> download link location)

[<EntryPoint>]
let main argv = 
    run ()
    0