Merge pull request #215 from boutproject/test-cache-inputs

Cache input data for tests

Author: johnomotani

setup.cfg

@@ -30,13 +30,13 @@ install_requires =
     xarray>=0.18.0
     boutdata>=0.1.4
     dask[array]>=2.10.0
+    gelidum>=0.5.3
     natsort>=5.5.0
     matplotlib>=3.1.1,!=3.3.0,!=3.3.1,!=3.3.2
     animatplot>=0.4.2
     netcdf4>=1.4.0
     Pillow>=6.1.0
     importlib-metadata; python_version < "3.8"
-tests_require = pytest >= 3.3.0
 include_package_data = True
 packages = find:
 

xbout/load.py

@@ -153,6 +153,10 @@ def open_boutdataset(
 
     if "reload" in input_type:
         if input_type == "reload":
+            if isinstance(datapath, Path):
+                # xr.open_mfdataset only accepts glob patterns as strings, not Path
+                # objects
+                datapath = str(datapath)
             ds = xr.open_mfdataset(
                 datapath,
                 chunks=chunks,

xbout/tests/test_against_collect.py

@@ -10,21 +10,21 @@
 
 
 class TestAccuracyAgainstOldCollect:
-    def test_single_file(self, tmpdir_factory):
+    def test_single_file(self, tmp_path_factory):
 
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         generated_ds = create_bout_ds(syn_data_type="linear")
-        generated_ds.to_netcdf(str(test_dir.join("BOUT.dmp.0.nc")))
+        generated_ds.to_netcdf(test_dir.joinpath("BOUT.dmp.0.nc"))
 
         var = "n"
         expected = old_collect(var, path=test_dir, xguards=True, yguards=False)
 
         # Test against new standard - open_boutdataset
         with pytest.warns(UserWarning):
-            ds = open_boutdataset(test_dir.join("BOUT.dmp.0.nc"))
+            ds = open_boutdataset(test_dir.joinpath("BOUT.dmp.0.nc"))
         actual = ds[var].values
 
         assert expected.shape == actual.shape
@@ -36,24 +36,24 @@ def test_single_file(self, tmpdir_factory):
         assert expected.shape == actual.shape
         npt.assert_equal(actual, expected)
 
-    def test_multiple_files_along_x(self, tmpdir_factory):
+    def test_multiple_files_along_x(self, tmp_path_factory):
 
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         ds_list, file_list = create_bout_ds_list(
             "BOUT.dmp", nxpe=3, nype=1, syn_data_type="linear"
         )
         for temp_ds, file_name in zip(ds_list, file_list):
-            temp_ds.to_netcdf(str(test_dir.join(str(file_name))))
+            temp_ds.to_netcdf(test_dir.joinpath(file_name))
 
         var = "n"
         expected = old_collect(var, path=test_dir, prefix="BOUT.dmp", xguards=True)
 
         # Test against new standard - open_boutdataset
         with pytest.warns(UserWarning):
-            ds = open_boutdataset(test_dir.join("BOUT.dmp.*.nc"))
+            ds = open_boutdataset(test_dir.joinpath("BOUT.dmp.*.nc"))
         actual = ds[var].values
 
         assert expected.shape == actual.shape
@@ -65,24 +65,24 @@ def test_multiple_files_along_x(self, tmpdir_factory):
         assert expected.shape == actual.shape
         npt.assert_equal(actual, expected)
 
-    def test_multiple_files_along_y(self, tmpdir_factory):
+    def test_multiple_files_along_y(self, tmp_path_factory):
 
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         ds_list, file_list = create_bout_ds_list(
             "BOUT.dmp", nxpe=1, nype=3, syn_data_type="linear"
         )
         for temp_ds, file_name in zip(ds_list, file_list):
-            temp_ds.to_netcdf(str(test_dir.join(str(file_name))))
+            temp_ds.to_netcdf(test_dir.joinpath(file_name))
 
         var = "n"
         expected = old_collect(var, path=test_dir, prefix="BOUT.dmp", xguards=True)
 
         # Test against new standard - .open_boutdataset
         with pytest.warns(UserWarning):
-            ds = open_boutdataset(test_dir.join("BOUT.dmp.*.nc"))
+            ds = open_boutdataset(test_dir.joinpath("BOUT.dmp.*.nc"))
         actual = ds[var].values
 
         assert expected.shape == actual.shape
@@ -94,24 +94,24 @@ def test_multiple_files_along_y(self, tmpdir_factory):
         assert expected.shape == actual.shape
         npt.assert_equal(actual, expected)
 
-    def test_multiple_files_along_xy(self, tmpdir_factory):
+    def test_multiple_files_along_xy(self, tmp_path_factory):
 
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         ds_list, file_list = create_bout_ds_list(
             "BOUT.dmp", nxpe=3, nype=3, syn_data_type="linear"
         )
         for temp_ds, file_name in zip(ds_list, file_list):
-            temp_ds.to_netcdf(str(test_dir.join(str(file_name))))
+            temp_ds.to_netcdf(test_dir.joinpath(file_name))
 
         var = "n"
         expected = old_collect(var, path=test_dir, prefix="BOUT.dmp", xguards=True)
 
         # Test against new standard - .open_boutdataset
         with pytest.warns(UserWarning):
-            ds = open_boutdataset(test_dir.join("BOUT.dmp.*.nc"))
+            ds = open_boutdataset(test_dir.joinpath("BOUT.dmp.*.nc"))
         actual = ds[var].values
 
         assert expected.shape == actual.shape
@@ -123,16 +123,16 @@ def test_multiple_files_along_xy(self, tmpdir_factory):
         assert expected.shape == actual.shape
         npt.assert_equal(actual, expected)
 
-    def test_metadata(self, tmpdir_factory):
+    def test_metadata(self, tmp_path_factory):
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         generated_ds = create_bout_ds(syn_data_type="linear")
-        generated_ds.to_netcdf(str(test_dir.join("BOUT.dmp.0.nc")))
+        generated_ds.to_netcdf(test_dir.joinpath("BOUT.dmp.0.nc"))
 
         with pytest.warns(UserWarning):
-            ds = open_boutdataset(test_dir.join("BOUT.dmp.*.nc"))
+            ds = open_boutdataset(test_dir.joinpath("BOUT.dmp.*.nc"))
 
         for v in METADATA_VARS:
             expected = old_collect(v, path=test_dir)
@@ -144,17 +144,17 @@ def test_metadata(self, tmpdir_factory):
             actual = new_collect(v, path=test_dir)
             npt.assert_equal(actual, expected)
 
-    def test_new_collect_indexing_int(self, tmpdir_factory):
+    def test_new_collect_indexing_int(self, tmp_path_factory):
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         ds_list, file_list = create_bout_ds_list(
             "BOUT.dmp", nxpe=3, nype=3, syn_data_type="linear"
         )
 
         for temp_ds, file_name in zip(ds_list, file_list):
-            temp_ds.to_netcdf(str(test_dir.join(str(file_name))))
+            temp_ds.to_netcdf(test_dir.joinpath(file_name))
 
         var = "n"
         indexers = ["tind", "xind", "yind", "zind"]
@@ -170,16 +170,16 @@ def test_new_collect_indexing_int(self, tmpdir_factory):
             assert expected.shape == actual.shape
             npt.assert_equal(actual, expected)
 
-    def test_new_collect_indexing_list(self, tmpdir_factory):
+    def test_new_collect_indexing_list(self, tmp_path_factory):
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         ds_list, file_list = create_bout_ds_list(
             "BOUT.dmp", nxpe=3, nype=3, syn_data_type="linear"
         )
         for temp_ds, file_name in zip(ds_list, file_list):
-            temp_ds.to_netcdf(str(test_dir.join(str(file_name))))
+            temp_ds.to_netcdf(test_dir.joinpath(file_name))
 
         var = "n"
         indexers = ["tind", "xind", "yind", "zind"]
@@ -195,17 +195,17 @@ def test_new_collect_indexing_list(self, tmpdir_factory):
             assert expected.shape == actual.shape
             npt.assert_equal(actual, expected)
 
-    def test_new_collect_indexing_slice(self, tmpdir_factory):
+    def test_new_collect_indexing_slice(self, tmp_path_factory):
         # Create temp directory for files
-        test_dir = tmpdir_factory.mktemp("test_data")
+        test_dir = tmp_path_factory.mktemp("test_data")
 
         # Generate some test data
         ds_list, file_list = create_bout_ds_list(
             "BOUT.dmp", nxpe=3, nype=3, syn_data_type="linear"
         )
 
         for temp_ds, file_name in zip(ds_list, file_list):
-            temp_ds.to_netcdf(str(test_dir.join(str(file_name))))
+            temp_ds.to_netcdf(test_dir.joinpath(file_name))
 
         var = "n"
         indexers = ["tind", "xind", "yind", "zind"]

xbout/tests/test_animate.py

@@ -12,20 +12,20 @@
 
 
 @pytest.fixture
-def create_test_file(tmpdir_factory):
+def create_test_file(tmp_path_factory):
 
     # Create temp dir for output of animate1D/2D
-    save_dir = tmpdir_factory.mktemp("test_data")
+    save_dir = tmp_path_factory.mktemp("test_data")
 
     # Generate some test data
     ds_list, file_list = create_bout_ds_list(
         "BOUT.dmp", nxpe=3, nype=3, syn_data_type="linear"
     )
     for ds, file_name in zip(ds_list, file_list):
-        ds.to_netcdf(str(save_dir.join(str(file_name))))
+        ds.to_netcdf(save_dir.joinpath(file_name))
 
     with pytest.warns(UserWarning):
-        ds = open_boutdataset(save_dir.join("BOUT.dmp.*.nc"))  # Open test data
+        ds = open_boutdataset(save_dir.joinpath("BOUT.dmp.*.nc"))  # Open test data
 
     return save_dir, ds