Skip to content

Commit 1697ded

Browse files
committed
Use xarray.open_mfdataset with better options
Using the same concatenation options as in custom implementation. Has essentially the same performance, much faster than original.
1 parent f925dba commit 1697ded

File tree

2 files changed

+18
-90
lines changed

2 files changed

+18
-90
lines changed

xbout/load.py

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -621,11 +621,20 @@ def _auto_open_mfboutdataset(
621621

622622
paths_grid, concat_dims = _arrange_for_concatenation(filepaths, nxpe, nype)
623623

624-
# Call custom implementation of open_mfdataset
625-
# avoiding some of the performance issues.
626-
from .mfdataset import mfdataset
627-
628-
ds = mfdataset(paths_grid, concat_dim=concat_dims, preprocess=_preprocess)
624+
ds = xr.open_mfdataset(
625+
paths_grid,
626+
concat_dim=concat_dims,
627+
combine="nested",
628+
data_vars="minimal", # Only data variables in which the dimension already appears are concatenated.
629+
coords = "minimal", # Only coordinates in which the dimension already appears are concatenated.
630+
compat = "override", # Duplicate data taken from first dataset
631+
combine_attrs="override", # Duplicate attributes taken from first dataset
632+
preprocess=_preprocess,
633+
engine=filetype,
634+
chunks=chunks,
635+
join="exact", # Don't align. Raise ValueError when indexes to be aligned are not equal
636+
**kwargs,
637+
)
629638
else:
630639
# datapath was nested list of Datasets
631640

@@ -669,7 +678,10 @@ def _auto_open_mfboutdataset(
669678
concat_dim=concat_dims,
670679
data_vars=data_vars,
671680
join="exact",
672-
combine_attrs="no_conflicts",
681+
data_vars="minimal", # Only data variables in which the dimension already appears are concatenated.
682+
coords = "minimal", # Only coordinates in which the dimension already appears are concatenated.
683+
compat = "override", # Duplicate data taken from first dataset
684+
combine_attrs="override", # Duplicate attributes taken from first dataset
673685
)
674686

675687
return ds, remove_yboundaries

xbout/mfdataset.py

Lines changed: 0 additions & 84 deletions
This file was deleted.

0 commit comments

Comments
 (0)