From 770955876d72f9de68dbc58c03f29da17d4d9c90 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Thu, 7 Nov 2024 09:52:04 -0500 Subject: [PATCH] fix metadata topic_hierarchy (#800) * fix metadata topic_hierarchy * test * test * test * fix --- .github/workflows/tests-docker.yml | 10 +++++----- docs/source/reference/quickstart.rst | 6 +++--- ...o-test-ship-hourly.yml => int-wmo-test-ship.yml} | 4 ++-- .../{wmo-test-ship-hourly.csv => wmo-test-ship.csv} | 0 ...LFPW111700RRA_C_EDZW_20240211170705_50982396.bin | Bin tests/integration/test_workflow.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) rename tests/data/metadata/discovery/{int-wmo-test-ship-hourly.yml => int-wmo-test-ship.yml} (97%) rename tests/data/metadata/station/{wmo-test-ship-hourly.csv => wmo-test-ship.csv} (100%) rename tests/data/observations/wmo/{ship-hourly => ship}/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin (100%) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 70ead8c5..dc73c6ab 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -147,11 +147,11 @@ jobs: python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add example hourly ship data (bufr2bufr) WMO env: - TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship-hourly - DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship-hourly.yml - DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship-hourly - STATION_METADATA: /data/wis2box/metadata/station/wmo-test-ship-hourly.csv - TEST_DATA: /data/wis2box/observations/wmo/ship-hourly + TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship.yml + DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship + STATION_METADATA: /data/wis2box/metadata/station/wmo-test-ship.csv + TEST_DATA: /data/wis2box/observations/wmo/ship run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst index f3f5fe5b..da6c6c05 100644 --- a/docs/source/reference/quickstart.rst +++ b/docs/source/reference/quickstart.rst @@ -54,7 +54,7 @@ Publish test datasets: wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/dz-surface-weather-observations.yml wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/ro-synoptic-weather-observations.yml wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/cd-surface-weather-observations.yml - wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-ship-hourly.yml + wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-ship.yml wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-drifting-buoys.yml wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-wind-profile.yml @@ -67,7 +67,7 @@ Load initial stations: wis2box metadata station publish-collection --path /data/wis2box/metadata/station/algeria.csv --topic-hierarchy origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop wis2box metadata station publish-collection --path /data/wis2box/metadata/station/romania.csv --topic-hierarchy origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop wis2box metadata station publish-collection --path /data/wis2box/metadata/station/congo.csv --topic-hierarchy origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-ship-hourly.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship-hourly + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-ship.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-drifting-buoys.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-wind-profile.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile @@ -80,7 +80,7 @@ Ingest data using the data ingest command to push data to the ``wis2box-incoming wis2box data ingest --metadata-id "urn:wmo:md:dz-meteoalgerie:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/algeria wis2box data ingest --metadata-id "urn:wmo:md:ro-rnimh-test:synoptic-weather-observations" --path $WIS2BOX_DATADIR/observations/romania wis2box data ingest --metadata-id "urn:wmo:md:cg-met:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/congo - wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:ship-hourly" --path $WIS2BOX_DATADIR/observations/wmo/ship-hourly + wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:ship" --path $WIS2BOX_DATADIR/observations/wmo/ship wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:drifting-buoys" --path $WIS2BOX_DATADIR/observations/wmo/drifting-buoys wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:wind-profile" --path $WIS2BOX_DATADIR/observations/wmo/wind-profile diff --git a/tests/data/metadata/discovery/int-wmo-test-ship-hourly.yml b/tests/data/metadata/discovery/int-wmo-test-ship.yml similarity index 97% rename from tests/data/metadata/discovery/int-wmo-test-ship-hourly.yml rename to tests/data/metadata/discovery/int-wmo-test-ship.yml index 0e5308e8..600db66b 100644 --- a/tests/data/metadata/discovery/int-wmo-test-ship-hourly.yml +++ b/tests/data/metadata/discovery/int-wmo-test-ship.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/ship-hourly + topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/ship country: int centre_id: int-wmo-test data_mappings: @@ -16,7 +16,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:ship-hourly + identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:ship hierarchylevel: dataset identification: diff --git a/tests/data/metadata/station/wmo-test-ship-hourly.csv b/tests/data/metadata/station/wmo-test-ship.csv similarity index 100% rename from tests/data/metadata/station/wmo-test-ship-hourly.csv rename to tests/data/metadata/station/wmo-test-ship.csv diff --git a/tests/data/observations/wmo/ship-hourly/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin b/tests/data/observations/wmo/ship/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin similarity index 100% rename from tests/data/observations/wmo/ship-hourly/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin rename to tests/data/observations/wmo/ship/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index c7f480c1..c27ad856 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -56,7 +56,7 @@ def test_wis2downloader(): 'origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop': 0, # noqa 'origin/a/wis2/int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys': 2, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile': 1, # noqa - 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship-hourly': 5, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa 'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1 # noqa }