Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add sub-hourly support to grib_to_netcdf #219

Open
wants to merge 10 commits into
base: develop
Choose a base branch
from
Open

Add sub-hourly support to grib_to_netcdf #219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7f07e7b
ECC-1755: Add sub hourly support
joobog Jun 14, 2024
8f4b8e6
ECC-1755: Add tests
joobog Jun 17, 2024
0f4d1f7
Merge branch 'develop' into bugfix/ecc-1755_grib_to_netcdf_sub_hourly
joobog Jun 17, 2024
57cb970
ECC-1755: Fix tests
shahramn Jun 17, 2024
13c2b40
ECC-1755: Cleanup
shahramn Jun 17, 2024
ad0072c
ECC-1755: Test issues (try 01)
shahramn Jun 17, 2024
36b2e1a
ECC-1755: Test issues (try 02)
shahramn Jun 17, 2024
d7bff50
ECC-1755: Test issues (try 03)
shahramn Jun 17, 2024
c41dfd2
ECC-1755: Test issues (try 04)
shahramn Jun 17, 2024
c1fcbf6
ECC-1755: Test issues (try 05)
shahramn Jun 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 56 additions & 29 deletions tests/grib_to_netcdf.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ fi

label="grib_to_netcdf_test"
tempGrib=temp.${label}.grib
tempGrib2=temp2.${label}.grib
tempNetcdf=temp.${label}.nc
tempText=temp.${label}.txt
tempDir=temp.${label}.dir
tempFilter=temp.${label}.filter

have_netcdf4=0

Expand All @@ -36,6 +38,59 @@ if command -v "ncdump" >/dev/null 2>&1; then
NC_DUMPER="ncdump"
fi

echo "Test ECC-1755: Test sub-hourly support ..."
# ------------------------------------------------------------
# ECC-1755: Sub-hourly support
# Minutes:
cat > $tempFilter <<EOF
transient steps={0,15,30};
meta step_long element(steps, count - 1);
meta step_string sprintf("%sm", step_long); # Seconds
set step = step_string;
write;
EOF

input=${data_dir}/sample.grib2
cat $input $input $input > $tempGrib
${tools_dir}/grib_filter -o $tempGrib2 $tempFilter $tempGrib
${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib2
if test "x$NC_DUMPER" != "x"; then
${NC_DUMPER} -t -v time $tempNetcdf > $tempText
cat $tempText
grep -q 'time:units = "minutes since 1900-01-01 00:00:00.0" ;' $tempText
grep -q 'time = "2008-02-06 12", "2008-02-06 12:15", "2008-02-06 12:30"' $tempText
fi

# Seconds:
cat > $tempFilter <<EOF
transient steps={0,15,30};
meta step_long element(steps, count - 1);
meta step_string sprintf("%ss", step_long); # Seconds
set step = step_string;
write;
EOF

input=${data_dir}/sample.grib2
cat $input $input $input > $tempGrib
${tools_dir}/grib_filter -o $tempGrib2 $tempFilter $tempGrib

# Please set the reference date to avoid an out-of-range error with the integer value.
${tools_dir}/grib_to_netcdf -R 20080206 -o $tempNetcdf $tempGrib2
if test "x$NC_DUMPER" != "x"; then
${NC_DUMPER} -t -v time $tempNetcdf > $tempText
cat $tempText
grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText
grep -q 'time = "2008-02-06 12' $tempText
grep -q '"2008-02-06 12:00:15' $tempText
grep -q '"2008-02-06 12:00:30' $tempText
fi

# The operation should fail because the time value exceeded the maximum limit of 2,147,483,647.
if ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib2; then
echo "Time values are out of range. Should fail."
exit 1
fi


echo "Test ECC-1041: One parameter with different expvers ..."
# ------------------------------------------------------------
Expand Down Expand Up @@ -110,14 +165,6 @@ ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib
${tools_dir}/grib_set -s productDefinitionTemplateNumber=31 $sample2 $tempGrib
${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib

ECCODES_DEBUG=-1 ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib


# The -u option
input=${data_dir}/sample.grib2
${tools_dir}/grib_to_netcdf -u time -o $tempNetcdf $input


echo "Test different resolutions ..."
# ------------------------------------
# This should fail as messages have different resolutions
Expand Down Expand Up @@ -178,26 +225,6 @@ set -e
grep -q "Wrong message length" $tempText


# Non-GRIB input
input=$data_dir/bufr/aaen_55.bufr
set +e
${tools_dir}/grib_to_netcdf -o $tempNetcdf $input > $tempText 2>&1
status=$?
set -e
[ $status -ne 0 ]
grep -q "Input does not contain any field" $tempText


# Bad reference date
input=$data_dir/sample.grib2
set +e
${tools_dir}/grib_to_netcdf -Rxxx -o $tempNetcdf $input > $tempText 2>&1
status=$?
set -e
[ $status -ne 0 ]
grep -q "Invalid reference date" $tempText


# Validity time check
export GRIB_TO_NETCDF_CHECKVALIDTIME=0
${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib
Expand All @@ -206,4 +233,4 @@ unset GRIB_TO_NETCDF_CHECKVALIDTIME


# Clean up
rm -f $tempNetcdf $tempGrib $tempText
rm -f $tempNetcdf $tempGrib $tempGrib2 $tempText $tempFilter
50 changes: 41 additions & 9 deletions tools/grib_to_netcdf.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,21 @@ static bool eq_time(const char* l, const char* r)
return false;
}

static const char* get_step_units_longname(const char* step_units)
{
if (step_units == NULL)
return NULL;
if (strcmp(step_units, "s") == 0)
return "seconds";
if (strcmp(step_units, "m") == 0)
return "minutes";
if (strcmp(step_units, "h") == 0)
return "hours";

Assert(!"Unknown step units");
return NULL;
}

static value* new_value(const char* name)
{
value* v = (value*)calloc(sizeof(value), 1);
Expand Down Expand Up @@ -1974,6 +1989,7 @@ static void validation_time(request* r)
double v;
long julian = 0;
const char* step_units = NULL;
double step_units_factor = 1.0;

long nstep = count_values(r, "step");
long ndate = count_values(r, "date");
Expand Down Expand Up @@ -2051,21 +2067,23 @@ static void validation_time(request* r)
"Cannot convert stepUnits of '%s'. Only hours, minutes and seconds supported.", step_units);
}
if (STR_EQUAL("m", step_units)) {
step /= 60;
step_units_factor = 60.0;
}
else if (STR_EQUAL("s", step_units)) {
step /= 3600;
step_units_factor = 3600.0;
}
}
v = julian * 24.0 + fcmonthdays * 24.0 + time / 100.0 + step * 1.0;

v = (julian * 24.0 + fcmonthdays * 24.0 + time / 100.0) * step_units_factor + step * 1.0;

grib_context_log(ctx, GRIB_LOG_DEBUG, "grib_to_netcdf: date=%ld, julian=%ld, fcmonthdays=%ld, time=%ld, step=%g, validation=%.3f", date, julian, fcmonthdays, time, step, v);
set_value(r, "_validation", "%lf", v);
set_value(r, "_juliandate", "%ld", julian);

if (!julianrefdate)
julianrefdate = grib_date_to_julian(setup.refdate);

set_value(r, "_validationtime", "%lf", v - julianrefdate * 24.0);
set_value(r, "_validationtime", "%lf", v - julianrefdate * 24.0 * step_units_factor);

/* Remove minutes from TIME */
if (ntime)
Expand Down Expand Up @@ -3007,12 +3025,15 @@ static int define_netcdf_dimensions(hypercube* h, fieldset* fs, int ncid, datase
}
}

const char* step_units = get_value(data_r, "stepUnits", 0);
const char* step_units_longname= get_step_units_longname(step_units);

if (strcmp(axis, "time") == 0) {
bool onedtime = (count_values(cube, "date") == 0 && count_values(cube, "step") == 0);
snprintf(u, sizeof(u), "hours since 0000-00-00 00:00:00.0");
snprintf(u, sizeof(u), "%s since 0000-00-00 00:00:00.0", step_units_longname);
longname = "reference_time";
if (setup.usevalidtime || onedtime) {
snprintf(u, sizeof(u), "hours since %ld-%02ld-%02ld 00:00:00.0", setup.refdate / 10000, (setup.refdate % 10000) / 100, (setup.refdate % 100));
snprintf(u, sizeof(u), "%s since %ld-%02ld-%02ld 00:00:00.0", step_units_longname, setup.refdate / 10000, (setup.refdate % 10000) / 100, (setup.refdate % 100));
longname = "time";
}
if (setup.climatology) {
Expand All @@ -3023,7 +3044,7 @@ static int define_netcdf_dimensions(hypercube* h, fieldset* fs, int ncid, datase
}

if (strcmp(axis, "step") == 0) {
units = "hours";
units = step_units;
longname = "time_step";
if (count_values(cube, "date") == 0 && count_values(cube, "time") == 0) {
const char* d = get_value(data_r, "date", 0);
Expand Down Expand Up @@ -3299,15 +3320,26 @@ static int fill_netcdf_dimensions(hypercube* h, fieldset* fs, int ncid)
for (j = 0; j < n; ++j)
values[j] = monthnumber(get_value(cube, axis, j));
else
for (j = 0; j < n; ++j)
values[j] = grib_date_to_julian(atol(get_value(cube, axis, j))) - grib_date_to_julian(setup.refdate);
for (j = 0; j < n; ++j) {
long date = grib_date_to_julian(atol(get_value(cube, axis, j)));
long refdate = grib_date_to_julian(setup.refdate);
fprintf(stderr, "date=%ld, refdate=%ld\n", date, refdate);
values[j] = date - refdate;
//values[j] = grib_date_to_julian(atol(get_value(cube, axis, j))) - grib_date_to_julian(setup.refdate);
}
}
else {
for (j = 0; j < n; ++j) {
long lv = 0;
const char* sv = get_value(cube, axis, j);
if (is_number(sv)) {
lv = atol(sv); /* Detect error? */
constexpr long maxint = 2147483647; /* 2^31 - 1, max size of NC_INT*/
if (lv > maxint) {
grib_context_log(ctx, GRIB_LOG_ERROR, "Value %ld for %s is too large. ", lv, axis);
grib_context_log(ctx, GRIB_LOG_ERROR, "Consider using the option: \"-R date\". Reference date in the format YYYYMMDD. Default value 19000101.", setup.refdate);
exit(1);
}
}
else {
/* ECC-725: Convert string-valued dimension to integer
Expand Down
Loading