Example using TileDB-R's batched reader API.

tiledb-batched-reads.R

#! /usr/bin/env Rscript

library(tiledb)
library(data.table)
library(nycflights13)

TILEDB_URI <- file.path(tempdir(), "tiledb-flights")
if (dir.exists(TILEDB_URI)) unlink(TILEDB_URI, recursive = TRUE)

STATS_DIR <- file.path(getwd(), "stats")
dir.create(STATS_DIR, recursive = TRUE, showWarnings = FALSE)

fromDataFrame(
  obj = flights,
  uri = TILEDB_URI,
  col_index = "time_hour",
  sparse = TRUE
)

# restrict memory to force batched reads
set_allocation_size_preference(1024^2)

# enable tiledb stats
tiledb_stats_enable()
tdb <- tiledb_array(TILEDB_URI, query_layout = "UNORDERED")

# create a batched query
batched <- createBatched(tdb)

flights2 <- list()
i <- 1

while (!completedBatched(batched)) {
  message("Batch ", i)
  flights2[[i]] <- fetchBatched(tdb, batched)
  tiledb_stats_dump(file.path(STATS_DIR, paste0("batch_", i, ".txt")))
  i <- i + 1
  # reset the stats after each query
  tiledb_stats_reset()
}

flights2 <- data.table::rbindlist(flights2)
flights2