Process results for GPM IMERG

import hvplot.pandas  # noqa
from utils import process_results
df = process_results("results/2024-10-25").sort_values(["duration (s)"])

Show memory and time for warp resampling dataset

df = df[df["task"] == "resample"]
df = df[df["dataset"] == "gpm_imerg"]
df[df["zoom"] == "0"].style.background_gradient(cmap="YlOrRd")
  dataset task format method driver virtual zoom peak memory (GB) duration (s)
12 gpm_imerg resample netcdf pyresample h5netcdf local 0 0.510845 2.825584
3 gpm_imerg resample netcdf odc h5netcdf local 0 0.219070 2.873800
51 gpm_imerg resample zarr rioxarray zarr icechunk 0 0.214933 3.248126
48 gpm_imerg resample zarr pyresample zarr icechunk 0 0.484800 3.420104
24 gpm_imerg resample netcdf rioxarray zarr icechunk 0 0.213210 3.432233
6 gpm_imerg resample netcdf odc zarr icechunk 0 0.218082 3.527490
45 gpm_imerg resample zarr odc zarr icechunk 0 0.219883 3.595781
15 gpm_imerg resample netcdf pyresample zarr icechunk 0 0.509712 3.631176
21 gpm_imerg resample netcdf rioxarray h5netcdf local 0 1.133411 4.130201
39 gpm_imerg resample netcdf xesmfcached h5netcdf local 0 0.424940 5.035078
54 gpm_imerg resample zarr sparse zarr icechunk 0 0.168040 6.213787
0 gpm_imerg resample netcdf odc h5netcdf 0 0.238359 8.207994
36 gpm_imerg resample netcdf xesmfcached h5netcdf 0 0.436165 9.760592
60 gpm_imerg resample zarr xesmfcached zarr icechunk 0 0.402360 10.216686
42 gpm_imerg resample netcdf xesmfcached zarr icechunk 0 0.402591 10.589594
18 gpm_imerg resample netcdf rioxarray h5netcdf 0 1.153169 10.647312
9 gpm_imerg resample netcdf pyresample h5netcdf 0 0.505636 11.401241
30 gpm_imerg resample netcdf xesmf h5netcdf local 0 1.001367 39.583973
57 gpm_imerg resample zarr xesmf zarr icechunk 0 0.997985 39.985007
33 gpm_imerg resample netcdf xesmf zarr icechunk 0 0.999235 41.595189
27 gpm_imerg resample netcdf xesmf h5netcdf 0 1.021253 48.698868 
df[df["zoom"] == "1"].style.background_gradient(cmap="YlOrRd")
  dataset task format method driver virtual zoom peak memory (GB) duration (s)
13 gpm_imerg resample netcdf pyresample h5netcdf local 1 0.238092 2.623957
4 gpm_imerg resample netcdf odc h5netcdf local 1 0.153546 2.728585
22 gpm_imerg resample netcdf rioxarray h5netcdf local 1 0.361287 2.744986
52 gpm_imerg resample zarr rioxarray zarr icechunk 1 0.140098 3.236913
25 gpm_imerg resample netcdf rioxarray zarr icechunk 1 0.158653 3.243511
49 gpm_imerg resample zarr pyresample zarr icechunk 1 0.211257 3.273793
46 gpm_imerg resample zarr odc zarr icechunk 1 0.144622 3.343807
7 gpm_imerg resample netcdf odc zarr icechunk 1 0.161910 3.438085
16 gpm_imerg resample netcdf pyresample zarr icechunk 1 0.228543 3.451255
40 gpm_imerg resample netcdf xesmfcached h5netcdf local 1 0.424828 4.936243
55 gpm_imerg resample zarr sparse zarr icechunk 1 0.167936 5.932873
19 gpm_imerg resample netcdf rioxarray h5netcdf 1 0.382223 8.025317
1 gpm_imerg resample netcdf odc h5netcdf 1 0.172499 8.083535
10 gpm_imerg resample netcdf pyresample h5netcdf 1 0.239817 8.201538
61 gpm_imerg resample zarr xesmfcached zarr icechunk 1 0.402356 10.472459
37 gpm_imerg resample netcdf xesmfcached h5netcdf 1 0.436163 10.723995
43 gpm_imerg resample netcdf xesmfcached zarr icechunk 1 0.402587 11.147670
31 gpm_imerg resample netcdf xesmf h5netcdf local 1 1.001367 38.868661
58 gpm_imerg resample zarr xesmf zarr icechunk 1 0.997985 39.246454
34 gpm_imerg resample netcdf xesmf zarr icechunk 1 0.999235 46.250185
28 gpm_imerg resample netcdf xesmf h5netcdf 1 1.021253 48.669996 
df[df["zoom"] == "2"].style.background_gradient(cmap="YlOrRd")
  dataset task format method driver virtual zoom peak memory (GB) duration (s)
23 gpm_imerg resample netcdf rioxarray h5netcdf local 2 0.138885 2.382297
14 gpm_imerg resample netcdf pyresample h5netcdf local 2 0.152566 2.642978
5 gpm_imerg resample netcdf odc h5netcdf local 2 0.152049 2.802723
53 gpm_imerg resample zarr rioxarray zarr icechunk 2 0.117588 3.109849
50 gpm_imerg resample zarr pyresample zarr icechunk 2 0.126922 3.223160
17 gpm_imerg resample netcdf pyresample zarr icechunk 2 0.161032 3.265925
26 gpm_imerg resample netcdf rioxarray zarr icechunk 2 0.158644 3.282040
47 gpm_imerg resample zarr odc zarr icechunk 2 0.121783 3.430303
8 gpm_imerg resample netcdf odc zarr icechunk 2 0.160676 3.491717
41 gpm_imerg resample netcdf xesmfcached h5netcdf local 2 0.423986 4.907868
56 gpm_imerg resample zarr sparse zarr icechunk 2 0.167899 5.912529
11 gpm_imerg resample netcdf pyresample h5netcdf 2 0.171652 8.141710
2 gpm_imerg resample netcdf odc h5netcdf 2 0.171469 8.328904
20 gpm_imerg resample netcdf rioxarray h5netcdf 2 0.159274 8.705396
62 gpm_imerg resample zarr xesmfcached zarr icechunk 2 0.402338 10.346199
44 gpm_imerg resample netcdf xesmfcached zarr icechunk 2 0.402595 10.624055
38 gpm_imerg resample netcdf xesmfcached h5netcdf 2 0.436163 10.637578
32 gpm_imerg resample netcdf xesmf h5netcdf local 2 1.001367 38.748044
59 gpm_imerg resample zarr xesmf zarr icechunk 2 0.997985 39.848156
35 gpm_imerg resample netcdf xesmf zarr icechunk 2 0.999366 42.124982
29 gpm_imerg resample netcdf xesmf h5netcdf 2 1.021253 45.680152