.serialize Returns an Error #666
Description
Hello there,
I am new to using intake-esm. I have been following the documentation to read data from a data catalog file and subset it to a new data catalog file.
To do this when I use the .serialize function, it does not write the .jsonc file telling that I do not have write access. ButI do have rite access to the location mentioned in the error message.
Can someone help me figure out this issue.
I tried using the method for NCAR managed data catalogs as well as the datastore url. Both returns the same error.
Thank you!
Nish
Below is the error that I recieve.
> ---------------------------------------------------------------------------
> FileNotFoundError Traceback (most recent call last)
> Cell In[5], line 1
> ----> 1 col_subset.serialize(directory='/glade/work/netige/mdtf_Apr24_2/data_catalogs', name="my_catalog_subset")
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:55, in validate_arguments.<locals>.validate.<locals>.wrapper_function(*args, **kwargs)
> 53 @wraps(_func)
> 54 def wrapper_function(*args: Any, **kwargs: Any) -> Any:
> ---> 55 return vd.call(*args, **kwargs)
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:150, in ValidatedFunction.call(self, *args, **kwargs)
> 148 def call(self, *args: Any, **kwargs: Any) -> Any:
> 149 m = self.init_model_instance(*args, **kwargs)
> --> 150 return self.execute(m)
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:222, in ValidatedFunction.execute(self, m)
> 220 return self.raw_function(*args_, **kwargs, **var_kwargs)
> 221 else:
> --> 222 return self.raw_function(**d, **var_kwargs)
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/intake_esm/core.py:492, in esm_datastore.serialize(self, name, directory, catalog_type, to_csv_kwargs, json_dump_kwargs, storage_options)
> 446 @pydantic.validate_arguments
> 447 def serialize(
> 448 self,
> (...)
> 454 storage_options: typing.Optional[dict[str, typing.Any]] = None,
> 455 ) -> None:
> 456 """Serialize catalog to corresponding json and csv files.
> 457
> 458 Parameters
> (...)
> 489 >>> cat_subset.serialize(name="cmip6_bcc_esm1", catalog_type="file")
> 490 """
> --> 492 self.esmcat.save(
> 493 name,
> 494 directory=directory,
> 495 catalog_type=catalog_type,
> 496 to_csv_kwargs=to_csv_kwargs,
> 497 json_dump_kwargs=json_dump_kwargs,
> 498 storage_options=storage_options,
> 499 )
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/intake_esm/cat.py:207, in ESMCatalogModel.save(self, name, directory, catalog_type, to_csv_kwargs, json_dump_kwargs, storage_options)
> 204 else:
> 205 data['catalog_dict'] = self.df.to_dict(orient='records')
> --> 207 with fs.open(json_file_name, 'w') as outfile:
> 208 json_kwargs = {'indent': 2}
> 209 json_kwargs.update(json_dump_kwargs or {})
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/spec.py:1295, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
> 1287 mode = mode.replace("t", "") + "b"
> 1289 text_kwargs = {
> 1290 k: kwargs.pop(k)
> 1291 for k in ["encoding", "errors", "newline"]
> 1292 if k in kwargs
> 1293 }
> 1294 return io.TextIOWrapper(
> -> 1295 self.open(
> 1296 path,
> 1297 mode,
> 1298 block_size=block_size,
> 1299 cache_options=cache_options,
> 1300 compression=compression,
> 1301 **kwargs,
> 1302 ),
> 1303 **text_kwargs,
> 1304 )
> 1305 else:
> 1306 ac = kwargs.pop("autocommit", not self._intrans)
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/spec.py:1307, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
> 1305 else:
> 1306 ac = kwargs.pop("autocommit", not self._intrans)
> -> 1307 f = self._open(
> 1308 path,
> 1309 mode=mode,
> 1310 block_size=block_size,
> 1311 autocommit=ac,
> 1312 cache_options=cache_options,
> 1313 **kwargs,
> 1314 )
> 1315 if compression is not None:
> 1316 from fsspec.compression import compr
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/implementations/local.py:180, in LocalFileSystem._open(self, path, mode, block_size, **kwargs)
> 178 if self.auto_mkdir and "w" in mode:
> 179 self.makedirs(self._parent(path), exist_ok=True)
> --> 180 return LocalFileOpener(path, mode, fs=self, **kwargs)
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/implementations/local.py:302, in LocalFileOpener.__init__(self, path, mode, autocommit, fs, compression, **kwargs)
> 300 self.compression = get_compression(path, compression)
> 301 self.blocksize = io.DEFAULT_BUFFER_SIZE
> --> 302 self._open()
>
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/implementations/local.py:307, in LocalFileOpener._open(self)
> 305 if self.f is None or self.f.closed:
> 306 if self.autocommit or "w" not in self.mode:
> --> 307 self.f = open(self.path, mode=self.mode)
> 308 if self.compression:
> 309 compress = compr[self.compression]
>
> FileNotFoundError: [Errno 2] No such file or directory: "/glade/work/netige/mdtf_Apr24_2/scripts/('file', 'local'):///glade/work/netige/mdtf_Apr24_2/data_catalogs/my_catalog_subset.json"