no message #990
GitHub Actions / Core / Unit Test Results (3.10)
failed
Oct 3, 2024 in 0s
23 fail, 675 pass in 7m 41s
698 tests 675 ✅ 7m 41s ⏱️
1 suites 0 💤
1 files 23 ❌
Results for commit df9dfd4.
Annotations
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_additional_variables (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3714c308e0>
fp = <http.client.HTTPResponse object at 0x7f37159c7310>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3714014100>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2017242N16333', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = ['numobs', 'season', 'number', 'subbasin', 'name', 'source_usa', ...]
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_additional_variables>
def test_ibtracs_additional_variables(self):
"""Check additional_variables option"""
# this is the complete list (as of May 2023) of variables in IBTrACS that are not
# agency-specific and that are not already considered by other parts of
# `from_ibtracs_netcdf`:
addtl_vars = [
'numobs', 'season', 'number', 'subbasin', 'name', 'source_usa', 'source_jma',
'source_cma', 'source_hko', 'source_new', 'source_reu', 'source_bom', 'source_nad',
'source_wel', 'source_td5', 'source_td6', 'source_ds8', 'source_neu', 'source_mlc',
'iso_time', 'nature', 'wmo_wind', 'wmo_pres', 'wmo_agency', 'track_type',
'main_track_sid', 'dist2land', 'landfall', 'iflag', 'storm_speed', 'storm_dir',
]
> tc_track = tc.TCTracks.from_ibtracs_netcdf(
storm_id='2017242N16333',
additional_variables=addtl_vars,
)
climada/hazard/test/test_tc_tracks.py:272:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2017242N16333', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = ['numobs', 'season', 'number', 'subbasin', 'name', 'source_usa', ...]
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_antimeridian (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37155d3be0>
fp = <http.client.HTTPResponse object at 0x7f37155d36a0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f37155d31c0>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = ['official_3h']
rescale_windspeeds = True, storm_id = '2013224N12220', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_antimeridian>
def test_ibtracs_antimeridian(self):
"""Read a track that crosses the antimeridian and make sure that lon is consistent"""
storm_id = '2013224N12220'
# the officially responsible agencies 'usa' and 'tokyo' use different signs in lon, but we
# have to `estimate_missing` because both have gaps in reported values
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storm_id, provider=['official_3h'],
estimate_missing=True)
climada/hazard/test/test_tc_tracks.py:152:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = ['official_3h']
rescale_windspeeds = True, storm_id = '2013224N12220', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_correct_pass (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3714ce98d0>
fp = <http.client.HTTPResponse object at 0x7f3714ceacb0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3714c8d450>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1982267N25289', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_correct_pass>
def test_ibtracs_correct_pass(self):
"""Check estimate_missing option"""
> tc_try = tc.TCTracks.from_ibtracs_netcdf(
provider='usa', storm_id='1982267N25289', estimate_missing=True)
climada/hazard/test/test_tc_tracks.py:240:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1982267N25289', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_discard_single_points (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f36be7016f0>
fp = <http.client.HTTPResponse object at 0x7f36be7021d0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f36be701f90>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = None, year_range = (1863, 1863)
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = False
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_discard_single_points>
def test_ibtracs_discard_single_points(self):
"""Check discard_single_points option"""
passed = False
for year in range(1863, 1981):
> tc_track_singlept = tc.TCTracks.from_ibtracs_netcdf(
provider='usa', year_range=(year,year), discard_single_points=False)
climada/hazard/test/test_tc_tracks.py:250:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = None, year_range = (1863, 1863)
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = False
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_estimate_missing (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3713a902b0>
fp = <http.client.HTTPResponse object at 0x7f3713a90bb0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f36be94d030>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_estimate_missing>
def test_ibtracs_estimate_missing(self):
"""Read a tropical cyclone and estimate missing values."""
storm_id = '2012152N12130'
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storm_id, estimate_missing=True)
climada/hazard/test/test_tc_tracks.py:161:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_interpolate_missing (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37159f03d0>
fp = <http.client.HTTPResponse object at 0x7f37159f24a0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f37159f3700>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2010066S19050', year_range = None
basin = None, genesis_basin = None, interpolate_missing = False
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_interpolate_missing>
def test_ibtracs_interpolate_missing(self):
"""Read a tropical cyclone with and without interpolating missing values."""
storm_id = '2010066S19050'
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storm_id, interpolate_missing=False)
climada/hazard/test/test_tc_tracks.py:208:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2010066S19050', year_range = None
basin = None, genesis_basin = None, interpolate_missing = False
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_official (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37159b5fc0>
fp = <http.client.HTTPResponse object at 0x7f37159b6ce0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f37159b6920>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'official'
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = False
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_official>
def test_ibtracs_official(self):
"""Read a tropical cyclone, only officially reported values."""
storm_id = '2012152N12130'
> tc_track = tc.TCTracks.from_ibtracs_netcdf(
storm_id=storm_id, interpolate_missing=False, provider='official')
climada/hazard/test/test_tc_tracks.py:183:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'official'
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = False
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_range (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3714fa9a80>
fp = <http.client.HTTPResponse object at 0x7f3714fa80d0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3714faa3b0>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = None, year_range = (2100, 2150)
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_range>
def test_ibtracs_range(self):
"""Read several TCs."""
> tc_track = tc.TCTracks.from_ibtracs_netcdf(year_range=(2100, 2150))
climada/hazard/test/test_tc_tracks.py:223:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = None, year_range = (2100, 2150)
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_raw_pass (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3715566350>
fp = <http.client.HTTPResponse object at 0x7f3715564910>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3715564520>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2017242N16333', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_raw_pass>
def test_ibtracs_raw_pass(self):
"""Read a tropical cyclone."""
# read without specified provider or estimation of missing values
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id='2017242N16333')
climada/hazard/test/test_tc_tracks.py:86:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2017242N16333', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_scale_wind (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37155d0850>
fp = <http.client.HTTPResponse object at 0x7f37155d0370>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f37155d03a0>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_scale_wind>
def test_ibtracs_scale_wind(self):
"""Read a tropical cyclone and scale wind speed according to agency."""
storm_id = '2012152N12130'
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storm_id, rescale_windspeeds=True)
climada/hazard/test/test_tc_tracks.py:196:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_ibtracs_with_provider (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37159cf2e0>
fp = <http.client.HTTPResponse object at 0x7f37159ccbe0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f37159cfd30>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_ibtracs_with_provider>
def test_ibtracs_with_provider(self):
"""Read a tropical cyclone with and without explicit provider."""
storm_id = '2012152N12130'
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storm_id, provider='usa')
climada/hazard/test/test_tc_tracks.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '2012152N12130', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_penv_rmax_penv_pass (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37155286a0>
fp = <http.client.HTTPResponse object at 0x7f371552a620>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f371552ba60>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1992230N11325', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_penv_rmax_penv_pass>
def test_penv_rmax_penv_pass(self):
"""from_ibtracs_netcdf"""
> tc_track = tc.TCTracks.from_ibtracs_netcdf(provider='usa', storm_id='1992230N11325')
climada/hazard/test/test_tc_tracks.py:74:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1992230N11325', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_raw_ibtracs_empty_pass (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3714ce98d0>
fp = <http.client.HTTPResponse object at 0x7f36be779c60>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f36be77a4d0>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1988234N13299', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_raw_ibtracs_empty_pass>
def test_raw_ibtracs_empty_pass(self):
"""Test reading empty TC from IBTrACS files"""
> tc_track = tc.TCTracks.from_ibtracs_netcdf(
provider='usa', storm_id='1988234N13299')
climada/hazard/test/test_tc_tracks.py:55:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1988234N13299', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIbtracs
github-actions / Core / Unit Test Results (3.10)
test_raw_ibtracs_invalid_pass (climada.hazard.test.test_tc_tracks.TestIbtracs) failed
tests_xml/tests.xml [took 0s]
Raw output
AssertionError: 'IDs are invalid' not found in 'Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc'
self = <climada.hazard.test.test_tc_tracks.TestIbtracs testMethod=test_raw_ibtracs_invalid_pass>
def test_raw_ibtracs_invalid_pass(self):
"""Test reading invalid/non-existing TC from IBTrACS files"""
with self.assertRaises(ValueError) as cm:
tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id='INVALID')
> self.assertIn("IDs are invalid", str(cm.exception))
E AssertionError: 'IDs are invalid' not found in 'Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc'
climada/hazard/test/test_tc_tracks.py:64: AssertionError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIO
github-actions / Core / Unit Test Results (3.10)
test_hdf5_io (climada.hazard.test.test_tc_tracks.TestIO) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37155665f0>
fp = <http.client.HTTPResponse object at 0x7f3714f9a800>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3714f9a740>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = None, year_range = (1993, 1994)
basin = 'EP', genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIO testMethod=test_hdf5_io>
def test_hdf5_io(self):
"""Test writing and reading hdf5 TCTracks instances"""
path = DATA_DIR.joinpath("tc_tracks.h5")
> tc_track = tc.TCTracks.from_ibtracs_netcdf(
provider='usa', year_range=(1993, 1994), basin='EP', estimate_missing=True)
climada/hazard/test/test_tc_tracks.py:325:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = None, year_range = (1993, 1994)
basin = 'EP', genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestIO
github-actions / Core / Unit Test Results (3.10)
test_netcdf_io (climada.hazard.test.test_tc_tracks.TestIO) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3713d540a0>
fp = <http.client.HTTPResponse object at 0x7f36be786c80>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f36be7873d0>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1988234N13299', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = ['numobs', 'storm_speed', 'nature']
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestIO testMethod=test_netcdf_io>
def test_netcdf_io(self):
"""Test writting and reading netcdf4 TCTracks instances"""
path = DATA_DIR.joinpath("tc_tracks_nc")
path.mkdir(exist_ok=True)
> tc_track = tc.TCTracks.from_ibtracs_netcdf(
provider='usa', storm_id='1988234N13299', estimate_missing=True,
additional_variables=["numobs", "storm_speed", "nature"],
)
climada/hazard/test/test_tc_tracks.py:297:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1988234N13299', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = True, correct_pres = False, discard_single_points = True
additional_variables = ['numobs', 'storm_speed', 'nature']
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestFuncs
github-actions / Core / Unit Test Results (3.10)
test_generate_centroids (climada.hazard.test.test_tc_tracks.TestFuncs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37159c64d0>
fp = <http.client.HTTPResponse object at 0x7f3714f741c0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3714f74160>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = ['usa', 'bom']
rescale_windspeeds = True
storm_id = ['1988169N14259', '2002073S16161', '2002143S07157']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestFuncs testMethod=test_generate_centroids>
def test_generate_centroids(self):
"""Test centroids generation feature."""
storms = ['1988169N14259', '2002073S16161', '2002143S07157']
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storms, provider=["usa", "bom"])
climada/hazard/test/test_tc_tracks.py:589:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = ['usa', 'bom']
rescale_windspeeds = True
storm_id = ['1988169N14259', '2002073S16161', '2002143S07157']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestFuncs
github-actions / Core / Unit Test Results (3.10)
test_get_extent (climada.hazard.test.test_tc_tracks.TestFuncs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3714f6bdf0>
fp = <http.client.HTTPResponse object at 0x7f3714f6a800>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3714f69720>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = ['usa', 'bom']
rescale_windspeeds = True
storm_id = ['1988169N14259', '2002073S16161', '2002143S07157']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestFuncs testMethod=test_get_extent>
def test_get_extent(self):
"""Test extent/bounds attributes."""
storms = ['1988169N14259', '2002073S16161', '2002143S07157']
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storms, provider=["usa", "bom"])
climada/hazard/test/test_tc_tracks.py:579:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = ['usa', 'bom']
rescale_windspeeds = True
storm_id = ['1988169N14259', '2002073S16161', '2002143S07157']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestFuncs
github-actions / Core / Unit Test Results (3.10)
test_subset (climada.hazard.test.test_tc_tracks.TestFuncs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37159c48b0>
fp = <http.client.HTTPResponse object at 0x7f37159ccdc0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f37159ce7d0>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True
storm_id = ['1988169N14259', '2002073S16161', '2002143S07157']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestFuncs testMethod=test_subset>
def test_subset(self):
"""Test subset."""
storms = ['1988169N14259', '2002073S16161', '2002143S07157']
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=storms)
climada/hazard/test/test_tc_tracks.py:573:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True
storm_id = ['1988169N14259', '2002073S16161', '2002143S07157']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks.TestFuncs
github-actions / Core / Unit Test Results (3.10)
test_tracks_in_exp_pass (climada.hazard.test.test_tc_tracks.TestFuncs) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3715108d30>
fp = <http.client.HTTPResponse object at 0x7f3715108e20>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f371510a950>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = ['2000233N12316', '2000160N21267']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks.TestFuncs testMethod=test_tracks_in_exp_pass>
def test_tracks_in_exp_pass(self):
"""Check if tracks in exp are filtered correctly"""
# Load two tracks from ibtracks
storms = {'in': '2000233N12316', 'out': '2000160N21267'}
> tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id=list(storms.values()))
climada/hazard/test/test_tc_tracks.py:898:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = ['2000233N12316', '2000160N21267']
year_range = None, basin = None, genesis_basin = None
interpolate_missing = True, estimate_missing = False, correct_pres = False
discard_single_points = True, additional_variables = None
file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks_synth.TestDecay
github-actions / Core / Unit Test Results (3.10)
test_wrong_decay_pass (climada.hazard.test.test_tc_tracks_synth.TestDecay) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f371510cb80>
fp = <http.client.HTTPResponse object at 0x7f371510ed10>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f371510e6e0>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1975178N28281', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks_synth.TestDecay testMethod=test_wrong_decay_pass>
def test_wrong_decay_pass(self):
"""Test decay not implemented when coefficient < 1"""
> track = tc.TCTracks.from_ibtracs_netcdf(provider='usa', storm_id='1975178N28281')
climada/hazard/test/test_tc_tracks_synth.py:293:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = '1975178N28281', year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_tc_tracks_synth.TestSynth
github-actions / Core / Unit Test Results (3.10)
test_random_walk_single_point (climada.hazard.test.test_tc_tracks_synth.TestSynth) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f37146aa9e0>
fp = <http.client.HTTPResponse object at 0x7f37146a98d0>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f37146aa770>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = None, year_range = (1951, 1951)
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = False
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_tc_tracks_synth.TestSynth testMethod=test_random_walk_single_point>
def test_random_walk_single_point(self):
found = False
for year in range(1951, 1981):
> tc_track = tc.TCTracks.from_ibtracs_netcdf(provider='usa',
year_range=(year,year),
discard_single_points=False)
climada/hazard/test/test_tc_tracks_synth.py:586:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = 'usa'
rescale_windspeeds = True, storm_id = None, year_range = (1951, 1951)
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = False
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Check warning on line 0 in climada.hazard.test.test_trop_cyclone.TestReader
github-actions / Core / Unit Test Results (3.10)
test_cross_antimeridian (climada.hazard.test.test_trop_cyclone.TestReader) failed
tests_xml/tests.xml [took 0s]
Raw output
ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
> urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
climada/util/files_handler.py:129:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:241: in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:216: in urlopen
return opener.open(url, data, timeout)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:525: in open
response = meth(req, response)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:634: in http_response
response = self.parent.error(
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:563: in error
return self._call_chain(*args)
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:496: in _call_chain
result = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib.request.HTTPDefaultErrorHandler object at 0x7f36c6b52560>
req = <urllib.request.Request object at 0x7f3714d24e20>
fp = <http.client.HTTPResponse object at 0x7f3714d24700>, code = 404
msg = 'Not Found', hdrs = <http.client.HTTPMessage object at 0x7f3714d24430>
def http_error_default(self, req, fp, code, msg, hdrs):
> raise HTTPError(req.full_url, code, msg, hdrs, fp)
E urllib.error.HTTPError: HTTP Error 404: Not Found
../../../micromamba/envs/climada_env_3.10/lib/python3.10/urllib/request.py:643: HTTPError
The above exception was the direct cause of the following exception:
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = ['2020346S13168'], year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
> download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
climada/hazard/tc_tracks.py:459:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc'
file_name = 'IBTrACS.ALL.v04r00.nc'
def download_ftp(url, file_name):
"""Download file from ftp in current folder.
Parameters
----------
url : str
url containing data to download
file_name : str
name of the file to dowload
Raises
------
ValueError
"""
LOGGER.info('Downloading file %s', file_name)
try:
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as prog_bar:
urllib.request.urlretrieve(url, file_name, reporthook=prog_bar.update_to)
except Exception as exc:
> raise ValueError(
f'{exc.__class__} - "{exc}": failed to retrieve {url} into {file_name}'
) from exc
E ValueError: <class 'urllib.error.HTTPError'> - "HTTP Error 404: Not Found": failed to retrieve https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf/IBTrACS.ALL.v04r00.nc into IBTrACS.ALL.v04r00.nc
climada/util/files_handler.py:131: ValueError
The above exception was the direct cause of the following exception:
self = <climada.hazard.test.test_trop_cyclone.TestReader testMethod=test_cross_antimeridian>
def test_cross_antimeridian(self):
# Two locations on the island Taveuni (Fiji), one west and one east of 180° longitude.
# We list the second point twice, with different lon-normalization:
cen = Centroids.from_lat_lon([-16.95, -16.8, -16.8], [179.9, 180.1, -179.9])
cen.set_dist_coast(precomputed=True)
# Cyclone YASA (2020) passed directly over Fiji
> tr = TCTracks.from_ibtracs_netcdf(storm_id=["2020346S13168"])
climada/hazard/test/test_trop_cyclone.py:134:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'climada.hazard.tc_tracks.TCTracks'>, provider = None
rescale_windspeeds = True, storm_id = ['2020346S13168'], year_range = None
basin = None, genesis_basin = None, interpolate_missing = True
estimate_missing = False, correct_pres = False, discard_single_points = True
additional_variables = None, file_name = 'IBTrACS.ALL.v04r00.nc'
@classmethod
def from_ibtracs_netcdf(cls, provider=None, rescale_windspeeds=True, storm_id=None,
year_range=None, basin=None, genesis_basin=None,
interpolate_missing=True, estimate_missing=False, correct_pres=False,
discard_single_points=True, additional_variables=None,
file_name='IBTrACS.ALL.v04r00.nc'):
"""Create new TCTracks object from IBTrACS databse.
When using data from IBTrACS, make sure to be familiar with the scope and limitations of
IBTrACS, e.g. by reading the official documentation
(https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf).
Reading the CLIMADA documentation can't replace a thorough understanding of the underlying
data. This function only provides a (hopefully useful) interface for the data input, but
cannot provide any guidance or make recommendations about if and how to use IBTrACS data
for your particular project.
Resulting tracks are required to have both pressure and wind speed information at all time
steps. Therefore, all track positions where one of wind speed or pressure are missing are
discarded unless one of `interpolate_missing` or `estimate_missing` are active.
Some corrections are automatically applied, such as: `environmental_pressure` is enforced
to be larger than `central_pressure`.
Note that the tracks returned by this function might contain irregular time steps since
that is often the case for the original IBTrACS records: many agencies add an additional
time step at landfall. Apply the `equal_timestep` function afterwards to enforce regular
time steps.
Parameters
----------
provider : str or list of str, optional
Either specify an agency, such as "usa", "newdelhi", "bom", "cma", "tokyo", or the
special values "official" and "official_3h":
* "official" means using the (usually 6-hourly) officially reported values of the
officially responsible agencies.
* "official_3h" means to include (inofficial) 3-hourly data of the officially
responsible agencies (whenever available).
If you want to restrict to the officially reported values by the officially responsible
agencies (`provider="official"`) without any modifications to the original official
data, make sure to also set `estimate_missing=False` and `interpolate_missing=False`.
Otherwise, gaps in the official reporting will be filled using interpolation and/or
statistical estimation procedures (see below).
If a list is given, the following logic is applied: For each storm, the variables that
are not reported by the first agency for this storm are taken from the next agency in
the list that did report this variable for this storm. For different storms, the same
variable might be taken from different agencies.
Default: ``['official_3h', 'usa', 'tokyo', 'newdelhi', 'reunion', 'bom', 'nadi',
'wellington', 'cma', 'hko', 'ds824', 'td9636', 'td9635', 'neumann', 'mlc']``
rescale_windspeeds : bool, optional
If True, all wind speeds are linearly rescaled to 1-minute sustained winds.
Note however that the IBTrACS documentation (Section 5.2,
https://www.ncei.noaa.gov/sites/default/files/2021-07/IBTrACS_version4_Technical_Details.pdf)
includes a warning about this kind of conversion: "While a multiplicative factor can
the numerical differences, there are procedural and observational differences between
agencies that can change through time, which confounds the simple multiplicative
factor." Default: True
storm_id : str or list of str, optional
IBTrACS ID of the storm, e.g. 1988234N13299, [1988234N13299, 1989260N11316].
year_range : tuple (min_year, max_year), optional
Year range to filter track selection. Default: None.
basin : str, optional
If given, select storms that have at least one position in the specified basin. This
allows analysis of a given basin, but also means that basin-specific track sets should
not be combined across basins since some storms will be in more than one set. If you
would like to select storms by their (unique) genesis basin instead, use the parameter
`genesis_basin`. For possible values (basin abbreviations), see the parameter
`genesis_basin`. If None, this filter is not applied. Default: None.
genesis_basin : str, optional
The basin where a TC is formed is not defined in IBTrACS. However, this filter option
allows to restrict to storms whose first valid eye position is in the specified basin,
which simulates the genesis location. Note that the resulting genesis basin of a
particular track may depend on the selected `provider` and on `estimate_missing`
because only the first *valid* eye position is considered. Possible values are 'NA'
(North Atlantic), 'SA' (South Atlantic), 'EP' (Eastern North Pacific, which includes
the Central Pacific region), 'WP' (Western North Pacific), 'SP' (South Pacific),
'SI' (South Indian), 'NI' (North Indian). If None, this filter is not applied.
Default: None.
interpolate_missing : bool, optional
If True, interpolate temporal reporting gaps within a variable (such as pressure, wind
speed, or radius) linearly if possible. Temporal interpolation is with respect to the
time steps defined in IBTrACS for a particular storm. No new time steps are added that
are not originally defined in IBTrACS.
For each time step with a missing value, this procedure is only able to fill in that
value if there are other time steps before and after this time step for which values
have been reported.
This procedure will be applied before the statistical estimations referred to
by `estimate_missing`. It is applied to all variables (eye position, wind speed,
environmental and central pressure, storm radius and radius of maximum winds).
Default: True
estimate_missing : bool, optional
For each fixed time step, estimate missing pressure, wind speed and radius using other
variables that are available at that time step.
The relationships between the variables are purely statistical. In comparison to
`interpolate_missing`, this procedure is able to estimate values for variables that
haven't been reported by any agency at any time step, as long as other variables are
available.
A typical example are storms before 1950, for which there are often no reported values
for pressure, but for wind speed. In this case, a rough statistical pressure-wind
relationship is applied to estimate the missing pressure values from the available
wind-speed values.
Make sure to set `rescale_windspeeds=True` when using this option because the
statistical relationships are calibrated using rescaled wind speeds.
Default: False
correct_pres : bool, optional
For backwards compatibility, alias for `estimate_missing`.
This is deprecated, use `estimate_missing` instead!
discard_single_points : bool, optional
Whether to discard tracks that consists of a single point. Recommended for full
compatiblity with other functions such as `equal_timesteps`. Default: True.
file_name : str, optional
Name of NetCDF file to be dowloaded or located at climada/data/system.
Default: 'IBTrACS.ALL.v04r00.nc'
additional_variables : list of str, optional
If specified, additional IBTrACS data variables are extracted, such as "nature" or
"storm_speed". Only variables that are not agency-specific are supported.
Default: None.
Returns
-------
tracks : TCTracks
TCTracks with data from IBTrACS
"""
if correct_pres:
LOGGER.warning("`correct_pres` is deprecated. "
"Use `estimate_missing` instead.")
estimate_missing = True
if estimate_missing and not rescale_windspeeds:
LOGGER.warning(
"Using `estimate_missing` without `rescale_windspeeds` is strongly discouraged!")
ibtracs_path = SYSTEM_DIR.joinpath(file_name)
if not ibtracs_path.is_file():
try:
download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
shutil.move(IBTRACS_FILE, ibtracs_path)
except ValueError as err:
> raise ValueError(
f'Error while downloading {IBTRACS_URL}. Try to download it manually and '
f'put the file in {ibtracs_path}') from err
E ValueError: Error while downloading https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r00/access/netcdf. Try to download it manually and put the file in /home/runner/climada/data/IBTrACS.ALL.v04r00.nc
climada/hazard/tc_tracks.py:462: ValueError
Loading