pickle_xarray.md

library(reticulate)
p <- c("polars", "h5netcdf", "fsspec", "aiohttp", "requests", "xarray", "dask")
py_require(p)
polars <- import("polars")
xarray <- import("xarray")

pp <-  arrow::read_parquet("https://projects.pawsey.org.au/idea-objects/idea-curated-objects.parquet")
d <-  pp |> dplyr::filter(Dataset == "oisst-avhrr-v02r01")
d <-  d |> dplyr::mutate(source = glue::glue("https://projects.pawsey.org.au/{Bucket}/{Key}"))
d0 <-  d |> dplyr::slice(0:10) |> dplyr::select("source")
src <- d0$source


ds <- xarray$open_mfdataset(src, engine = "h5netcdf", parallel  = TRUE)
py_save_object(ds, "myds.pk")


## this round trips!!!
library(reticulate)
py_load_object("myds.pk")
<xarray.Dataset> Size: 166MB
Dimensions:  (time: 10, zlev: 1, lat: 720, lon: 1440)
Coordinates:
  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88
* lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9
* time     (time) datetime64[ns] 80B 1981-09-01T12:00:00 ... 1981-09-10T12:...
* zlev     (zlev) float32 4B 0.0
Data variables:
  anom     (time, zlev, lat, lon) float32 41MB dask.array<chunksize=(1, 1, 720, 1440), meta=np.ndarray>
  err      (time, zlev, lat, lon) float32 41MB dask.array<chunksize=(1, 1, 720, 1440), meta=np.ndarray>
  ice      (time, zlev, lat, lon) float32 41MB dask.array<chunksize=(1, 1, 720, 1440), meta=np.ndarray>
  sst      (time, zlev, lat, lon) float32 41MB dask.array<chunksize=(1, 1, 720, 1440), meta=np.ndarray>
  Attributes: (12/37)
title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...
id:                         oisst-avhrr-v02r01.19810901.nc

b <- ds$sst$values
ximage::ximage(b[1,1,720:1,])