Elixir program that fetches all Eve Online market data and writes to csv

orders.ex

Mix.install([
  {:req, "~> 0.5.0"},
  {:csv, "~> 3.2.2"}
])

regions = Req.get!("https://esi.evetech.net/v1/universe/regions").body
data = regions |> Task.async_stream(fn region ->

    response = "https://esi.evetech.net/v1/markets/#{region}/orders/"
    |> IO.inspect()
    |> Req.get!()

    pages = response.headers["x-pages"]
    |> List.first()
    |> String.to_integer()

    page_urls = if pages > 1 do
      for page <- 2..pages do "https://esi.evetech.net/v1/markets/#{region}/orders?page=#{page}" end
    else [] end

    delay = fn n -> trunc(Integer.pow(2, n) * 1000 * (1 - 0.1 * :rand.uniform())) end
    pages_data = page_urls
    |> Task.async_stream(fn url ->
      url
      |> IO.inspect()
      |> Req.get!(retry_delay: delay)
      |> Map.get(:body)
    end, timeout: 69_000)
    |> Enum.flat_map(fn {:ok, result} -> result end)

    orders = response.body
    orders ++ pages_data

end, max_concurrency: 69, timeout: 420_000)
|> Enum.flat_map(fn {:ok, result} -> result end)

headers = [
  "duration",
  "is_buy_order",
  "issued",
  "location_id",
  "min_volume",
  "order_id",
  "price",
  "range",
  "system_id",
  "type_id",
  "volume_remain",
  "volume_total"
]

csv_stream = CSV.encode(data, headers: headers)
csv_binary = csv_stream
  |> Enum.to_list()
  |> IO.iodata_to_binary()

File.write!("orders.csv", csv_binary)