check WCMP2 compliance with pywcmp, update to valid centre-ids

wmo-im · Aug 21, 2024 · 519ce72 · 519ce72
1 parent 8e7429a
commit 519ce72
Show file tree

Hide file tree

Showing 16 changed files with 88 additions and 94 deletions.
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
@@ -16,11 +16,10 @@ jobs:
       working-directory: tests
       run: |
         pip3 install -r requirements.txt
-        pip3 install check-jsonschema
     - name: cache schemas 📦
       run: |
         pywis-pubsub schema sync
-        curl -s https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json
+        pywcmp bundle sync
     - name: display Docker and Python versions 📦
       run: |
         docker version
@@ -55,18 +54,18 @@ jobs:
         curl -X POST http://localhost/wis2downloader/subscriptions -H "Content-Type: application/json" -H "Authorization: Bearer github123" -d @test.json
     - name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼
       env:
-        TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop
+        TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop
         TERRITORY: MWI
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations
+        DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations
         STATION_METADATA: /data/wis2box/metadata/station/malawi.csv
         TEST_DATA: /data/wis2box/observations/malawi
         TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA_UPDATE
     - name: add Italy synop data (bufr2bufr) 🇮🇹
@@ -80,8 +79,8 @@ jobs:
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: add Algeria synop data (bufr2bufr) 🇩🇿
       env:
@@ -94,22 +93,22 @@ jobs:
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
-    - name: add Romania synop data  (synop2bufr and csv2bufr aws-template) 🇷🇴
+    - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴
       env:
-        TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop
+        TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop
         TERRITORY: ROU
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh:synoptic-weather-observations
+        DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh-test:synoptic-weather-observations
         STATION_METADATA: /data/wis2box/metadata/station/romania.csv
         TEST_DATA: /data/wis2box/observations/romania
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: add Congo synop data (synop2bufr) 🇨🇩
       env:
@@ -122,8 +121,8 @@ jobs:
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID  # commented out until recommended data licensing is supported
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: add example ship data (bufr2bufr) WMO
       env:
@@ -135,8 +134,8 @@ jobs:
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: add example buoy data (bufr2bufr) WMO
       env:
@@ -148,8 +147,8 @@ jobs:
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: add example wind profiler data (bufr2bufr) WMO
       env:
@@ -161,8 +160,8 @@ jobs:
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: add China GRIB2 data (universal pipeline) 🇨🇳
       env:
@@ -171,8 +170,8 @@ jobs:
         TEST_DATA: /data/wis2box/observations/china
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: sleep 30 seconds then run integration tests ⚙️
       run: |

diff --git a/docs/source/reference/auth.rst b/docs/source/reference/auth.rst
@@ -44,7 +44,7 @@ first time a token is generated for a dataset
 
 .. code-block:: bash
 
-    wis2box auth add-token --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations mytoken
+    wis2box auth add-token --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations mytoken
 
 
 If no token is provided, a random string will be generated. Be sure to the record token now, there is no
@@ -58,8 +58,8 @@ Token credentials can be validated using the wis2box command line utility.
 .. code-block:: bash
 
     wis2box auth show
-    wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations mytoken
-    wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations notmytoken
+    wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations mytoken
+    wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations notmytoken
 
 
 Once a token has been generated, access to any data of that topic in the WAF or API requires token authentication.
@@ -68,8 +68,8 @@ easily added to requests using `cURL`_.
 
 .. code-block:: bash
 
-    curl -H "Authorization: Bearer mytoken" "http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop"
-    curl -H "Authorization: Bearer notmytoken" "http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop"
+    curl -H "Authorization: Bearer mytoken" "http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop"
+    curl -H "Authorization: Bearer notmytoken" "http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop"
 
 
 Removing Access Control
@@ -80,7 +80,7 @@ deleting individual tokens, or all tokens for a given topic hierarchy.
 
 .. code-block:: bash
 
-    wis2box auth remove-tokens --topic-hierarchy mw-mw_met_centre.data.core.weather.surface-based-observations.synop
+    wis2box auth remove-tokens --topic-hierarchy mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop
     wis2box auth show
 
 

diff --git a/docs/source/reference/data-access/python-api-owslib.ipynb b/docs/source/reference/data-access/python-api-owslib.ipynb
@@ -69,11 +69,11 @@
     "for dataset in collections['collections']:\n",
     "    print(dataset['title'])\n",
     "\n",
-    "malawi_obs = oafeat.collection_items('mw-mw_met_centre.data.core.weather.surface-based-observations.synop')\n",
+    "malawi_obs = oafeat.collection_items('mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop')\n",
     "malawi_obs_df = pd.DataFrame(malawi_obs['features'])\n",
     "\n",
     "# then filter by station\n",
-    "obs = oafeat.collection_items('mw-mw_met_centre.data.core.weather.surface-based-observations.synop', wigos_station_identifier='0-454-2-AWSCHIDOOLE', name='air_temperature', limit=10000)\n",
+    "obs = oafeat.collection_items('mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop', wigos_station_identifier='0-454-2-AWSCHIDOOLE', name='air_temperature', limit=10000)\n",
     "\n",
     "datestamp = [obs['properties']['resultTime'] for obs in obs['features']]\n",
     "air_temperature = [obs['properties']['value'] for obs in obs['features']]\n",

diff --git a/docs/source/reference/data-access/python-api-requests.ipynb b/docs/source/reference/data-access/python-api-requests.ipynb
@@ -119,7 +119,7 @@
       "Datasets:\n",
       "\n",
       "id: data.core.test-passthrough, title: Surface weather observations (passthrough)\n",
-      "id: mw-mw_met_centre.data.core.weather.surface-based-observations.synop, title: Surface weather observations (hourly)\n"
+      "id: mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop, title: Surface weather observations (hourly)\n"
      ]
     }
    ],
@@ -151,18 +151,18 @@
      "text": [
       "Data access links:\n",
       "\n",
-      "{'rel': 'self', 'type': 'application/geo+json', 'title': 'This document as GeoJSON', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json (application/geo+json) self\n",
-      "{'rel': 'alternate', 'type': 'application/ld+json', 'title': 'This document as RDF (JSON-LD)', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld (application/ld+json) alternate\n",
-      "{'rel': 'alternate', 'type': 'text/html', 'title': 'This document as HTML', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html (text/html) alternate\n",
+      "{'rel': 'self', 'type': 'application/geo+json', 'title': 'This document as GeoJSON', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=json'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=json (application/geo+json) self\n",
+      "{'rel': 'alternate', 'type': 'application/ld+json', 'title': 'This document as RDF (JSON-LD)', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=jsonld'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=jsonld (application/ld+json) alternate\n",
+      "{'rel': 'alternate', 'type': 'text/html', 'title': 'This document as HTML', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=html'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=html (text/html) alternate\n",
       "{'rel': 'collection', 'type': 'application/json', 'title': 'Discovery metadata', 'href': 'http://localhost/oapi/collections/discovery-metadata'} http://localhost/oapi/collections/discovery-metadata (application/json) collection\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "['http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json',\n",
-       " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld',\n",
-       " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html',\n",
+       "['http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=json',\n",
+       " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=jsonld',\n",
+       " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop?f=html',\n",
        " 'http://localhost/oapi/collections/discovery-metadata']"
       ]
      },
@@ -172,7 +172,7 @@
     }
    ],
    "source": [
-    "dataset_id = 'mw-mw_met_centre.data.core.weather.surface-based-observations.synop'\n",
+    "dataset_id = 'mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop'\n",
     "\n",
     "url = f\"{api}/collections/discovery-metadata/items/{dataset_id}\"\n",
     "\n",
@@ -201,7 +201,7 @@
     {
      "data": {
       "text/plain": [
-       "'http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop'"
+       "'http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop'"
       ]
      },
      "execution_count": 5,
@@ -210,7 +210,7 @@
     }
    ],
    "source": [
-    "dataset_api_link = 'http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop'\n",
+    "dataset_api_link = 'http://localhost/oapi/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop'\n",
     "\n",
     "dataset_api_link"
    ]

diff --git a/docs/source/reference/data-access/r-api.ipynb b/docs/source/reference/data-access/r-api.ipynb
@@ -203,7 +203,7 @@
     }
    ],
    "source": [
-    "malawi_obs <- read_sf(paste0(oapi,\"/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items\"))\n",
+    "malawi_obs <- read_sf(paste0(oapi,\"/collections/mw-mw_met_centre-test.data.core.weather.surface-based-observations.synop/items\"))\n",
     "print(malawi_obs)"
    ]
   },