From 519ce72871751f776583a14f8c0ff9071c1a35f8 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 21 Aug 2024 17:02:30 -0400 Subject: [PATCH] check WCMP2 compliance with pywcmp, update to valid centre-ids --- .github/workflows/tests-docker.yml | 49 +++++++++---------- docs/source/reference/auth.rst | 12 ++--- .../data-access/python-api-owslib.ipynb | 4 +- .../data-access/python-api-requests.ipynb | 20 ++++---- docs/source/reference/data-access/r-api.ipynb | 2 +- docs/source/reference/quickstart.rst | 32 ++++++------ docs/source/reference/services.rst | 16 +++--- docs/source/reference/storage.rst | 4 +- docs/source/user/setup.rst | 2 +- .../metadata/discovery/int-wmo-test-buoy.yml | 4 +- .../metadata/discovery/int-wmo-test-ship.yml | 4 +- .../discovery/int-wmo-test-wind_profiler.yml | 4 +- .../mw-surface-weather-observations.yml | 6 +-- .../ro-synoptic-weather-observations.yml | 6 +-- tests/integration/test_workflow.py | 16 +++--- tests/requirements.txt | 1 + 16 files changed, 88 insertions(+), 94 deletions(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 98291017..c11d1ade 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -16,11 +16,10 @@ jobs: working-directory: tests run: | pip3 install -r requirements.txt - pip3 install check-jsonschema - name: cache schemas 📦 run: | pywis-pubsub schema sync - curl -s https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json + pywcmp bundle sync - name: display Docker and Python versions 📦 run: | docker version @@ -55,18 +54,18 @@ jobs: curl -X POST http://localhost/wis2downloader/subscriptions -H "Content-Type: application/json" -H "Authorization: Bearer github123" -d @test.json - name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼 env: - TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop + TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop TERRITORY: MWI DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml - DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations + DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations STATION_METADATA: /data/wis2box/metadata/station/malawi.csv TEST_DATA: /data/wis2box/observations/malawi TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA_UPDATE - name: add Italy synop data (bufr2bufr) 🇮🇹 @@ -80,8 +79,8 @@ jobs: run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add Algeria synop data (bufr2bufr) 🇩🇿 env: @@ -94,22 +93,22 @@ jobs: run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴 + - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴 env: - TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop + TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop TERRITORY: ROU DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml - DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh:synoptic-weather-observations + DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh-test:synoptic-weather-observations STATION_METADATA: /data/wis2box/metadata/station/romania.csv TEST_DATA: /data/wis2box/observations/romania run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add Congo synop data (synop2bufr) 🇨🇩 env: @@ -122,8 +121,8 @@ jobs: run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID # commented out until recommended data licensing is supported python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add example ship data (bufr2bufr) WMO env: @@ -135,8 +134,8 @@ jobs: run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add example buoy data (bufr2bufr) WMO env: @@ -148,8 +147,8 @@ jobs: run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add example wind profiler data (bufr2bufr) WMO env: @@ -161,8 +160,8 @@ jobs: run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add China GRIB2 data (universal pipeline) 🇨🇳 env: @@ -171,8 +170,8 @@ jobs: TEST_DATA: /data/wis2box/observations/china run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: sleep 30 seconds then run integration tests ⚙️ run: | diff --git a/docs/source/reference/auth.rst b/docs/source/reference/auth.rst index 8d9d920c..3156ed83 100644 --- a/docs/source/reference/auth.rst +++ b/docs/source/reference/auth.rst @@ -44,7 +44,7 @@ first time a token is generated for a dataset .. code-block:: bash - wis2box auth add-token --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations mytoken + wis2box auth add-token --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations mytoken If no token is provided, a random string will be generated. Be sure to the record token now, there is no @@ -58,8 +58,8 @@ Token credentials can be validated using the wis2box command line utility. .. code-block:: bash wis2box auth show - wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations mytoken - wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations notmytoken + wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations mytoken + wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations notmytoken Once a token has been generated, access to any data of that topic in the WAF or API requires token authentication. @@ -68,8 +68,8 @@ easily added to requests using `cURL`_. .. code-block:: bash - curl -H "Authorization: Bearer mytoken" "http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop" - curl -H "Authorization: Bearer notmytoken" "http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop" + curl -H "Authorization: Bearer mytoken" "http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop" + curl -H "Authorization: Bearer notmytoken" "http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop" Removing Access Control @@ -80,7 +80,7 @@ deleting individual tokens, or all tokens for a given topic hierarchy. .. code-block:: bash - wis2box auth remove-tokens --topic-hierarchy mw-mw_met_centre.data.core.weather.surface-based-observations.synop + wis2box auth remove-tokens --topic-hierarchy mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop wis2box auth show diff --git a/docs/source/reference/data-access/python-api-owslib.ipynb b/docs/source/reference/data-access/python-api-owslib.ipynb index 02c3cf5f..e991b253 100644 --- a/docs/source/reference/data-access/python-api-owslib.ipynb +++ b/docs/source/reference/data-access/python-api-owslib.ipynb @@ -69,11 +69,11 @@ "for dataset in collections['collections']:\n", " print(dataset['title'])\n", "\n", - "malawi_obs = oafeat.collection_items('mw-mw_met_centre.data.core.weather.surface-based-observations.synop')\n", + "malawi_obs = oafeat.collection_items('mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop')\n", "malawi_obs_df = pd.DataFrame(malawi_obs['features'])\n", "\n", "# then filter by station\n", - "obs = oafeat.collection_items('mw-mw_met_centre.data.core.weather.surface-based-observations.synop', wigos_station_identifier='0-454-2-AWSCHIDOOLE', name='air_temperature', limit=10000)\n", + "obs = oafeat.collection_items('mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop', wigos_station_identifier='0-454-2-AWSCHIDOOLE', name='air_temperature', limit=10000)\n", "\n", "datestamp = [obs['properties']['resultTime'] for obs in obs['features']]\n", "air_temperature = [obs['properties']['value'] for obs in obs['features']]\n", diff --git a/docs/source/reference/data-access/python-api-requests.ipynb b/docs/source/reference/data-access/python-api-requests.ipynb index 625372ad..fdc885e7 100644 --- a/docs/source/reference/data-access/python-api-requests.ipynb +++ b/docs/source/reference/data-access/python-api-requests.ipynb @@ -119,7 +119,7 @@ "Datasets:\n", "\n", "id: data.core.test-passthrough, title: Surface weather observations (passthrough)\n", - "id: mw-mw_met_centre.data.core.weather.surface-based-observations.synop, title: Surface weather observations (hourly)\n" + "id: mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop, title: Surface weather observations (hourly)\n" ] } ], @@ -151,18 +151,18 @@ "text": [ "Data access links:\n", "\n", - "{'rel': 'self', 'type': 'application/geo+json', 'title': 'This document as GeoJSON', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json (application/geo+json) self\n", - "{'rel': 'alternate', 'type': 'application/ld+json', 'title': 'This document as RDF (JSON-LD)', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld (application/ld+json) alternate\n", - "{'rel': 'alternate', 'type': 'text/html', 'title': 'This document as HTML', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html (text/html) alternate\n", + "{'rel': 'self', 'type': 'application/geo+json', 'title': 'This document as GeoJSON', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=json'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=json (application/geo+json) self\n", + "{'rel': 'alternate', 'type': 'application/ld+json', 'title': 'This document as RDF (JSON-LD)', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=jsonld'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=jsonld (application/ld+json) alternate\n", + "{'rel': 'alternate', 'type': 'text/html', 'title': 'This document as HTML', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=html'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=html (text/html) alternate\n", "{'rel': 'collection', 'type': 'application/json', 'title': 'Discovery metadata', 'href': 'http://localhost/oapi/collections/discovery-metadata'} http://localhost/oapi/collections/discovery-metadata (application/json) collection\n" ] }, { "data": { "text/plain": [ - "['http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json',\n", - " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld',\n", - " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html',\n", + "['http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=json',\n", + " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=jsonld',\n", + " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=html',\n", " 'http://localhost/oapi/collections/discovery-metadata']" ] }, @@ -172,7 +172,7 @@ } ], "source": [ - "dataset_id = 'mw-mw_met_centre.data.core.weather.surface-based-observations.synop'\n", + "dataset_id = 'mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop'\n", "\n", "url = f\"{api}/collections/discovery-metadata/items/{dataset_id}\"\n", "\n", @@ -201,7 +201,7 @@ { "data": { "text/plain": [ - "'http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop'" + "'http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop'" ] }, "execution_count": 5, @@ -210,7 +210,7 @@ } ], "source": [ - "dataset_api_link = 'http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop'\n", + "dataset_api_link = 'http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop'\n", "\n", "dataset_api_link" ] diff --git a/docs/source/reference/data-access/r-api.ipynb b/docs/source/reference/data-access/r-api.ipynb index d82c219d..48b2f524 100644 --- a/docs/source/reference/data-access/r-api.ipynb +++ b/docs/source/reference/data-access/r-api.ipynb @@ -203,7 +203,7 @@ } ], "source": [ - "malawi_obs <- read_sf(paste0(oapi,\"/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items\"))\n", + "malawi_obs <- read_sf(paste0(oapi,\"/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/items\"))\n", "print(malawi_obs)" ] }, diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst index 88b06501..1fbe7320 100644 --- a/docs/source/reference/quickstart.rst +++ b/docs/source/reference/quickstart.rst @@ -62,27 +62,27 @@ Load initial stations: .. code-block:: bash - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/malawi.csv --topic-hierarchy mw-mw_met_centre.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/italy.csv --topic-hierarchy it-meteoam.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/algeria.csv --topic-hierarchy dz-meteoalgerie.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/romania.csv --topic-hierarchy ro-rnimh.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/congo.csv --topic-hierarchy cg-met.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-ship.csv --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.ship - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-buoy.csv --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.buoy - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-wind-profiler.csv --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.wind_profiler + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/malawi.csv --topic-hierarchy origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/italy.csv --topic-hierarchy origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/algeria.csv --topic-hierarchy origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/romania.csv --topic-hierarchy origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/congo.csv --topic-hierarchy origin/a/wis2/cg-met/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-ship.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-buoy.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-wind_profiler.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler Ingest data using the data ingest command to push data to the ``wis2box-incoming`` bucket: .. code-block:: bash - wis2box data ingest --topic-hierarchy mw-mw_met_centre.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/malawi - wis2box data ingest --topic-hierarchy it-meteoam.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/italy - wis2box data ingest --topic-hierarchy dz-meteoalgerie.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/algeria - wis2box data ingest --topic-hierarchy ro-rnimh.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/romania - wis2box data ingest --topic-hierarchy cg-met.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/congo - wis2box data ingest --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.ship --path $WIS2BOX_DATADIR/observations/wmo/ship - wis2box data ingest --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.buoy --path $WIS2BOX_DATADIR/observations/wmo/buoy - wis2box data ingest --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.wind_profiler --path $WIS2BOX_DATADIR/observations/wmo/wind_profiler + wis2box data ingest --metadata-id "urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/malawi + wis2box data ingest --metadata-id "urn:wmo:md:it-meteoam:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/italy + wis2box data ingest --metadata-id "urn:wmo:md:dz-meteoalgerie:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/algeria + wis2box data ingest --metadata-id "urn:wmo:md:ro-rnimh-test:synoptic-weather-observations" --path $WIS2BOX_DATADIR/observations/romania + wis2box data ingest --metadata-id "urn:wmo:md:cg-met:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/congo + wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:ship" --path $WIS2BOX_DATADIR/observations/wmo/ship + wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:buoy" --path $WIS2BOX_DATADIR/observations/wmo/buoy + wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:wind_profiler" --path $WIS2BOX_DATADIR/observations/wmo/wind_profiler Logout of wis2box-management container: diff --git a/docs/source/reference/services.rst b/docs/source/reference/services.rst index 96bd99df..af879f69 100644 --- a/docs/source/reference/services.rst +++ b/docs/source/reference/services.rst @@ -45,21 +45,21 @@ Below are some examples of working with the discovery catalogue. .. note:: - - the examples below use the ``mw-mw_met_centre.data.core.weather.surface-based-observations.synop`` collection as described + - the examples below use the ``mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop`` collection as described in the :ref:`quickstart`. For other dataset collections, use the same query patterns below, substituting the collection id accordingly - list of dataset collections: http://localhost/oapi/collections -- collection description: http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop -- collection queryables: http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/queryables -- collection items (browse): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items +- collection description: http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop +- collection queryables: http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/queryables +- collection items (browse): http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/items - collection queries - - set limit/offset (paging): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?limit=1&startindex=2 - - query by spatial (bounding box): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?bbox=32,-17,36,-8 - - query by temporal extent (since): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?datetime=2021/.. - - query by temporal extent (before): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?datetime=../2022 + - set limit/offset (paging): http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/items?limit=1&startindex=2 + - query by spatial (bounding box): http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/items?bbox=32,-17,36,-8 + - query by temporal extent (since): http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/items?datetime=2021/.. + - query by temporal extent (before): http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/items?datetime=../2022 .. note:: diff --git a/docs/source/reference/storage.rst b/docs/source/reference/storage.rst index 125be200..32af6cb1 100644 --- a/docs/source/reference/storage.rst +++ b/docs/source/reference/storage.rst @@ -67,7 +67,7 @@ The below example copies a local file (``myfile.csv``) to the ``wis2box-incoming To allow uploading files into MinIO remotely, the ``wis2box-incoming`` bucket is proxied via Nginx. For example, to upload the local file (``WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv with topic``) to the folder -``mw-mw_met_centre/data/core/weather/surface-based-observations/synop``: +``mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop``: .. code-block:: python @@ -83,7 +83,7 @@ For example, to upload the local file (``WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0 s3client = session.client('s3', endpoint_url=endpoint_url) - folder = 'mw-mw_met_centre/data/core/weather/surface-based-observations/synop' + folder = 'mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop' with open(filename, 'rb') as fh: s3client.upload_fileobj(fh, 'wis2box-incoming', f'{folder}/{filename}') diff --git a/docs/source/user/setup.rst b/docs/source/user/setup.rst index f77bdcc1..57c7b6b0 100644 --- a/docs/source/user/setup.rst +++ b/docs/source/user/setup.rst @@ -268,7 +268,7 @@ You can also bulk insert a set of stations from a CSV file, by defining the stat .. code-block:: bash python3 wis2box-ctl.py login - wis2box metadata station publish-collection --path /data/wis2box/mystations.csv --topic-hierarchy origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/mystations.csv --topic-hierarchy origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop .. note:: diff --git a/tests/data/metadata/discovery/int-wmo-test-buoy.yml b/tests/data/metadata/discovery/int-wmo-test-buoy.yml index e439731d..db05763d 100644 --- a/tests/data/metadata/discovery/int-wmo-test-buoy.yml +++ b/tests/data/metadata/discovery/int-wmo-test-buoy.yml @@ -50,7 +50,7 @@ identification: url: https://example.org/fixme contact: - pointOfContact: &contact_poc + host: &contact_poc organization: World Meteorological Organisation url: https://www.wmo.int/ individualname: Firstname Lastname @@ -65,5 +65,3 @@ contact: email: null hoursofservice: 0800h - 1600h UTC contactinstructions: email - - distributor: *contact_poc diff --git a/tests/data/metadata/discovery/int-wmo-test-ship.yml b/tests/data/metadata/discovery/int-wmo-test-ship.yml index 56a37695..63b0fbda 100644 --- a/tests/data/metadata/discovery/int-wmo-test-ship.yml +++ b/tests/data/metadata/discovery/int-wmo-test-ship.yml @@ -50,7 +50,7 @@ identification: url: https://example.org/fixme contact: - pointOfContact: &contact_poc + host: &contact_poc organization: World Meteorological Organisation url: https://www.wmo.int/ individualname: Firstname Lastname @@ -65,5 +65,3 @@ contact: email: you@example.org hoursofservice: 0800h - 1600h UTC contactinstructions: email - - distributor: *contact_poc diff --git a/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml b/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml index e6eeacd3..082a6e01 100644 --- a/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml +++ b/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml @@ -47,7 +47,7 @@ identification: url: https://example.org/fixme contact: - pointOfContact: &contact_poc + host: &contact_poc organization: World Meteorological Organisation url: https://www.wmo.int/ individualname: Firstname Lastname @@ -62,5 +62,3 @@ contact: email: you@example.org hoursofservice: 0800h - 1600h UTC contactinstructions: email - - distributor: *contact_poc diff --git a/tests/data/metadata/discovery/mw-surface-weather-observations.yml b/tests/data/metadata/discovery/mw-surface-weather-observations.yml index bfd80d25..b7c3da6c 100644 --- a/tests/data/metadata/discovery/mw-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/mw-surface-weather-observations.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: mw-mw_met_centre/data/core/weather/surface-based-observations/synop + topic_hierarchy: mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop country: mwi - centre_id: mw-mw_met_centre + centre_id: mw-mw_met_centre-test data_mappings: plugins: csv: @@ -18,7 +18,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:mw-mw_met_centre:surface-weather-observations + identifier: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations hierarchylevel: dataset identification: diff --git a/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml b/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml index e55d791b..bd1a5973 100644 --- a/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml +++ b/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: ro-rnimh/data/core/weather/surface-based-observations/synop + topic_hierarchy: ro-rnimh-test/data/core/weather/surface-based-observations/synop country: rou - centre_id: ro-rnimh + centre_id: ro-rnimh-test data_mappings: plugins: txt: @@ -19,7 +19,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:ro-rnimh:synoptic-weather-observations + identifier: urn:wmo:md:ro-rnimh-test:synoptic-weather-observations hierarchylevel: dataset identification: diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index a1ef35a9..5a1e7581 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -35,7 +35,7 @@ URL = 'http://localhost' API_URL = f'{URL}/oapi' -ID = 'urn:wmo:md:mw-mw_met_centre:surface-weather-observations' +ID = 'urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations' SESSION = Session() SESSION.hooks = { 'response': lambda r, *args, **kwargs: r.raise_for_status() @@ -49,10 +49,10 @@ def test_wis2downloader(): DOWNLOAD_DIR = DATADIR / 'downloads' topic_nfiles_dict = { - 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop': 23, # noqa + 'origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop': 23, # noqa 'origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop': 28, # noqa 'origin/a/wis2/cn-cma/data/core/weather/prediction/forecast/medium-range/probabilistic/global': 10, # noqa - 'origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop': 49, # noqa + 'origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop': 49, # noqa 'origin/a/wis2/cg-met/data/core/weather/surface-based-observations/synop': 14, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy': 2, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler': 1, # noqa @@ -150,7 +150,7 @@ def test_metadata_discovery_publish(): mqtt_link = [d for d in r['links'] if d['href'].startswith('mqtt')][0] assert 'everyone:everyone' in mqtt_link['href'] - assert mqtt_link['channel'] == 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop' # noqa + assert mqtt_link['channel'] == 'origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop' # noqa params = { 'q': 'temperature' @@ -164,10 +164,10 @@ def test_metadata_discovery_publish(): # test access of discovery metadata from notification message centre_ids = [ - 'mw-mw_met_centre', + 'mw-mw_met_centre-test', 'it-meteoam', 'dz-meteoalgerie', - 'ro-rnimh', + 'ro-rnimh-test', 'cg-met', 'int-wmo-test' ] @@ -291,7 +291,7 @@ def test_message_api(): 'dz-meteoalgerie': 29, 'ro-rnimh': 50, 'cg-met': 15, - 'int-wmo-test': 11, + 'int-wmo': 11, 'cn-cma': 11 } for key, value in counts.items(): @@ -327,7 +327,7 @@ def test_message_api(): assert props['integrity']['method'] == 'sha512' assert not props['data_id'].startswith('wis2') assert not props['data_id'].startswith('origin/a/wis2') - assert props['data_id'].startswith('cd') + assert props['data_id'].startswith('cg') assert props['content']['size'] == 253 assert props['content']['encoding'] == 'base64' assert props['content']['value'] is not None diff --git a/tests/requirements.txt b/tests/requirements.txt index b6be1d99..3e58e592 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ flake8 pytest +pywcmp pywis-pubsub requests