countmap.jl

using NetCDF, YAXArrays
import DiskArrayEngine as DAE

ds = open_dataset("/Net/Groups/BGI/data/DataStructureMDI/DATA/Incoming/ESACCI/LC/v2_1_1/orgdata/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.nc")
lc = ds.lccs_class

c = YAXArrays.counter(lc)

using Distributed
addprocs(16)
@everywhere begin
    using Pkg; Pkg.activate(@__DIR__)
    using NetCDF, YAXArrays, Distributed
end
@time res = compute(c,runner=DAE.DaggerRunner)

countmap.py

from flox.xarray import xarray_reduce
import dask
from dask.distributed import Client, LocalCluster

cluster = LocalCluster(n_workers=16,threads_per_worker=1)
client = Client(cluster)

import xarray as xr
ds = xr.open_dataset("/Net/Groups/BGI/data/DataStructureMDI/DATA/Incoming/ESACCI/LC/v2_1_1/orgdata/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.nc")
                
#Need to chunk manually, otherwise it breaks
lc = ds.lccs_class.chunk(lon=2000,lat=2000)

a = xr.DataArray(dask.array.ones(lc.shape),dims=lc.dims,coords=lc.coords)

res = xarray_reduce(a,lc,func="sum",expected_groups=range(255)).compute()