Skip to content

Commit 591a226

Browse files
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
1 parent 0989d25 commit 591a226

File tree

4 files changed

+119
-115
lines changed

4 files changed

+119
-115
lines changed

disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ weather_code_nws:
1515
long_name: Weather code NWS
1616
units: ""
1717
alarms:
18-
description: PWS alarms flags
19-
long_name: PWS alarms flags
18+
description: PWS alarms flags
19+
long_name: PWS alarms flags
2020
units: ""
2121
sensor_status:
2222
description: Sensor status
@@ -65,7 +65,7 @@ raw_drop_number:
6565
type_distribution:
6666
description: Particle type distribution
6767
long_name: Particle type distribution
68-
units: ""
68+
units: ""
6969
drop_size_distribution:
7070
description: Drop size distribution
7171
long_name: Drop size distribution
@@ -74,4 +74,3 @@ peak_to_pedestal_hist:
7474
description: Peak to pedestal histogram
7575
long_name: Peak to pedestal histogram
7676
units: ""
77-

disdrodb/l0/configs/PWS100/l0b_encodings.yml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ alarms:
3939
shuffle: true
4040
fletcher32: false
4141
contiguous: false
42-
chunksizes: 5000
42+
chunksizes: 5000
4343
sensor_status:
4444
dtype: uint8
4545
zlib: true
@@ -68,7 +68,7 @@ relative_humidity:
6868
shuffle: true
6969
fletcher32: false
7070
contiguous: false
71-
chunksizes: 5000
71+
chunksizes: 5000
7272
air_temperature:
7373
dtype: uint16
7474
scale_factor: 0.01
@@ -148,7 +148,7 @@ drop_size_distribution:
148148
shuffle: true
149149
fletcher32: false
150150
contiguous: false
151-
chunksizes: 5000
151+
chunksizes: 5000
152152
peak_to_pedestal_hist:
153153
dtype: str
154154
zlib: false
@@ -164,7 +164,7 @@ type_distribution:
164164
shuffle: true
165165
fletcher32: false
166166
contiguous: false
167-
chunksizes: 5000
167+
chunksizes: 5000
168168
average_drop_velocity:
169169
dtype: uint16
170170
scale_factor: 0.01
@@ -185,4 +185,3 @@ average_drop_size:
185185
contiguous: false
186186
chunksizes: 5000
187187
_FillValue: 65535
188-

disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py

Lines changed: 41 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@
1717
# along with this program. If not, see <http://www.gnu.org/licenses/>.
1818
# -----------------------------------------------------------------------------.
1919
"""DISDRODB reader for ENPC PARSIVEL2 raw text data."""
20+
import zipfile
21+
2022
import numpy as np
2123
import pandas as pd
22-
import zipfile
23-
from disdrodb.utils.logger import log_error
24+
2425
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
2526
from disdrodb.l0.l0a_processing import read_raw_text_file
27+
from disdrodb.utils.logger import log_error
2628

2729

2830
@is_documented_by(reader_generic_docstring)
@@ -31,13 +33,14 @@ def reader(
3133
logger=None,
3234
):
3335
"""Reader."""
36+
3437
##------------------------------------------------------------------------.
3538
#### Define function to read each txt file inside each daily zip file
36-
def read_txt_file(file, filename):
39+
def read_txt_file(file, filename):
3740
##------------------------------------------------------------------------.
3841
#### Define column names
3942
column_names = ["TO_PARSE"]
40-
43+
4144
##------------------------------------------------------------------------.
4245
#### Define reader options
4346
reader_kwargs = {}
@@ -62,43 +65,43 @@ def read_txt_file(file, filename):
6265
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
6366
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
6467
reader_kwargs["na_values"] = ["na", "", "error"]
65-
66-
##------------------------------------------------------------------------.
68+
69+
##------------------------------------------------------------------------.
6770
#### Read the data
6871
df = read_raw_text_file(
6972
filepath=f,
7073
column_names=column_names,
7174
reader_kwargs=reader_kwargs,
7275
logger=logger,
7376
)
74-
77+
7578
##------------------------------------------------------------------------.
7679
#### Adapt the dataframe to adhere to DISDRODB L0 standards
7780
# Create ID and Value columns
7881
df = df["TO_PARSE"].str.split(":", expand=True, n=1)
79-
df.columns = ["ID", "Value"]
80-
82+
df.columns = ["ID", "Value"]
83+
8184
# Select only rows with values
8285
df = df[df["Value"].apply(lambda x: x is not None)]
83-
86+
8487
# Drop rows with invalid IDs
8588
valid_id_str = np.char.rjust(np.arange(0, 94).astype(str), width=2, fillchar="0")
8689
df = df[df["ID"].astype(str).isin(valid_id_str)]
87-
90+
8891
# Create the dataframe with each row corresponding to a timestep
8992
# - Group rows based on when ID values restart
9093
groups = df.groupby((df["ID"].astype(int).diff() <= 0).cumsum())
91-
94+
9295
# Reshape the dataframe
9396
group_dfs = []
9497
for _, group in groups:
9598
group_df = group.set_index("ID").T
9699
group_dfs.append(group_df)
97-
100+
98101
# Merge each timestep dataframe
99102
# --> Missing columns are infilled by NaN
100103
df = pd.concat(group_dfs, axis=0)
101-
104+
102105
# Assign column names
103106
column_dict = {
104107
"01": "rainfall_rate_32bit",
@@ -132,39 +135,39 @@ def read_txt_file(file, filename):
132135
"30": "rainfall_rate_16_bit_30",
133136
"31": "rainfall_rate_16_bit_1200",
134137
"32": "rainfall_accumulated_16bit",
135-
"34": "rain_kinetic_energy",
138+
"34": "rain_kinetic_energy",
136139
"35": "snowfall_rate",
137140
"90": "raw_drop_concentration",
138141
"91": "raw_drop_average_velocity",
139142
"93": "raw_drop_number",
140143
}
141-
144+
142145
df = df.rename(column_dict, axis=1)
143-
146+
144147
# Keep only columns defined in the dictionary
145148
df = df[list(column_dict.values())]
146-
147-
# Define datetime "time" column from filename
148-
datetime_str = ' '.join(filename.replace('.txt', '').split('_')[-6:])
149-
df["time"] = pd.to_datetime(datetime_str, format='%Y %m %d %H %M %S')
150-
149+
150+
# Define datetime "time" column from filename
151+
datetime_str = " ".join(filename.replace(".txt", "").split("_")[-6:])
152+
df["time"] = pd.to_datetime(datetime_str, format="%Y %m %d %H %M %S")
153+
151154
# # Drop columns not agreeing with DISDRODB L0 standards
152155
# columns_to_drop = [
153156
# "sensor_date",
154157
# "sensor_time",
155-
# "firmware_iop",
156-
# "firmware_dsp",
157-
# "sensor_serial_number",
158-
# "station_name",
159-
# "station_number",
158+
# "firmware_iop",
159+
# "firmware_dsp",
160+
# "sensor_serial_number",
161+
# "station_name",
162+
# "station_number",
160163
# ]
161164
# df = df.drop(columns=columns_to_drop)
162-
return df
163-
164-
#---------------------------------------------------------------------.
165+
return df
166+
167+
# ---------------------------------------------------------------------.
165168
#### Iterate over all files (aka timesteps) in the daily zip archive
166169
# - Each file contain a single timestep !
167-
list_df = []
170+
list_df = []
168171
with zipfile.ZipFile(filepath, "r") as zip_ref:
169172
filenames = sorted(zip_ref.namelist())
170173
for filename in filenames:
@@ -174,13 +177,12 @@ def read_txt_file(file, filename):
174177
try:
175178
df = read_txt_file(file=f, filename=filename)
176179
list_df.append(df)
177-
except Exception as e:
178-
msg = f"An error occured while reading {filename}. The error is: {e}."
180+
except Exception as e:
181+
msg = f"An error occured while reading {filename}. The error is: {e}."
179182
log_error(logger=logger, msg=msg, verbose=True)
180-
183+
181184
# Concatenate all dataframes into a single one
182-
df = pd.concat(list_df)
183-
184-
#---------------------------------------------------------------------.
185-
return df
186-
185+
df = pd.concat(list_df)
186+
187+
# ---------------------------------------------------------------------.
188+
return df

0 commit comments

Comments
 (0)