Merge branch 'feat-reload-official' of https://github.com/federal-geospatial-platform/pygeoapi into feat-reload-official

Author: Alex-NRCan

.github/workflows/containers.yml

@@ -25,7 +25,7 @@ jobs:
       contents: read
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@master
 
       - name: Set up QEMU
         uses: docker/[email protected]

.github/workflows/docs.yml

@@ -23,8 +23,8 @@ jobs:
         include:
           - python-version: '3.10'
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@master
+    - uses: actions/setup-python@v5
       name: Setup Python ${{ matrix.python-version }}
       with:
         python-version: ${{ matrix.python-version }}

.github/workflows/flake8.yml

@@ -7,8 +7,8 @@ jobs:
   flake8_py3:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@master
+      - uses: actions/setup-python@v5
         name: setup Python
         with:
           python-version: '3.10'

.github/workflows/main.yml

@@ -37,8 +37,8 @@ jobs:
     - name: Chown user
       run: |
         sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@master
+    - uses: actions/setup-python@v5
       name: Setup Python ${{ matrix.python-version }}
       with:
         python-version: ${{ matrix.python-version }}
@@ -56,7 +56,7 @@ jobs:
     - name: Install and run Elasticsearch 📦
       uses: getong/[email protected]
       with:
-        elasticsearch version: '8.3.1'
+        elasticsearch version: '8.17.0'
         host port: 9200
         container port: 9200
         host node port: 9300
@@ -65,25 +65,25 @@ jobs:
     - name: Install and run OpenSearch 📦
       uses: esmarkowski/[email protected]
       with:
-        version: 2.12.0
+        version: 2.18.0
         security-disabled: true
         port: 9209
     - name: Install and run MongoDB
-      uses: supercharge/mongodb-github-action@1.5.0
+      uses: supercharge/mongodb-github-action@1.12.0
       with:
-        mongodb-version: 4.4
+        mongodb-version: '8.0.4'
     - name: Install and run SensorThingsAPI
       uses: cgs-earth/[email protected]
     - name: Install sqlite and gpkg dependencies
-      uses: awalsh128/cache-apt-pkgs-action@latest
+      uses: awalsh128/cache-apt-pkgs-action@v1.4.3
       with:
         packages: libsqlite3-mod-spatialite
         version: 4.3.0a-6build1
     - name: Use ubuntuGIS unstable ppa
       run: sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable && sudo apt update
       shell: bash
     - name: Install GDAL with Python bindings
-      uses: awalsh128/cache-apt-pkgs-action@latest
+      uses: awalsh128/cache-apt-pkgs-action@v1.4.3
       with:
         packages: gdal-bin libgdal-dev
         version: 3.8.4

docker/default.config.yml

@@ -46,7 +46,9 @@ server:
     cors: true
     pretty_print: true
     admin: ${PYGEOAPI_SERVER_ADMIN:-false}
-    limit: 10
+    limits:
+        default_items: 10
+        max_items: 50
     # templates: /path/to/templates
     map:
         url: https://tile.openstreetmap.org/{z}/{x}/{y}.png

docs/source/configuration.rst

@@ -49,7 +49,15 @@ For more information related to API design rules (the ``api_rules`` property in
     gzip: false # default server config to gzip/compress responses to requests with gzip in the Accept-Encoding header
     cors: true  # boolean on whether server should support CORS
     pretty_print: true  # whether JSON responses should be pretty-printed
-    limit: 10  # server limit on number of items to return
+
+    limits:  # server limits on number of items to return.  This property can also be defined at the resource level to override global server settings
+        default_items: 50
+        max_items: 1000
+        max_distance_x: 25
+        max_distance_y: 25
+        max_distance_units: m
+        on_exceed: throttle  # throttle or error (default=throttle)
+
     admin: false  # whether to enable the Admin API
 
     # optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates
@@ -254,6 +262,41 @@ default.
 .. seealso::
    :ref:`plugins` for more information on plugins
 
+Using environment variables
+---------------------------
+
+pygeoapi configuration supports using system environment variables, which can be helpful
+for deploying into `12 factor <https://12factor.net/>`_ environments for example.
+
+Below is an example of how to integrate system environment variables in pygeoapi.
+
+.. code-block:: yaml
+
+   server:
+       bind:
+           host: ${MY_HOST}
+           port: ${MY_PORT}
+
+Multiple environment variables are supported as follows:
+
+.. code-block:: yaml
+
+   data: ${MY_HOST}:${MY_PORT}
+
+It is also possible to define a default value for a variable in case it does not exist in
+the environment using a syntax like: ``value: ${ENV_VAR:-the default}``
+
+.. code-block:: yaml
+
+   server:
+       bind:
+           host: ${MY_HOST:-localhost}
+           port: ${MY_PORT:-5000}
+   metadata:
+       identification:
+           title:
+               en: This is pygeoapi host ${MY_HOST} and port ${MY_PORT:-5000}, nice to meet you!
+
 Adding links to collections
 ---------------------------
 
@@ -389,53 +432,6 @@ If omitted, no header will be added. Common names for this header are ``API-Vers
 Note that pygeoapi already adds a ``X-Powered-By`` header by default that includes the software version number.
 
 
-Validating the configuration
-----------------------------
-
-To ensure your configuration is valid, pygeoapi provides a validation
-utility that can be run as follows:
-
-.. code-block:: bash
-
-   pygeoapi config validate -c /path/to/my-pygeoapi-config.yml
-
-
-Using environment variables
----------------------------
-
-pygeoapi configuration supports using system environment variables, which can be helpful
-for deploying into `12 factor <https://12factor.net/>`_ environments for example.
-
-Below is an example of how to integrate system environment variables in pygeoapi.
-
-.. code-block:: yaml
-
-   server:
-       bind:
-           host: ${MY_HOST}
-           port: ${MY_PORT}
-
-Multiple environment variables are supported as follows:
-
-.. code-block:: yaml
-
-   data: ${MY_HOST}:${MY_PORT}
-
-It is also possible to define a default value for a variable in case it does not exist in
-the environment using a syntax like: ``value: ${ENV_VAR:-the default}``
-
-.. code-block:: yaml
-
-   server:
-       bind:
-           host: ${MY_HOST:-localhost}
-           port: ${MY_PORT:-5000}
-   metadata:
-       identification:
-           title:
-               en: This is pygeoapi host ${MY_HOST} and port ${MY_PORT:-5000}, nice to meet you!
-
-
 Hierarchical collections
 ------------------------
 
@@ -507,6 +503,36 @@ Examples:
   curl https://example.org/collections/lakes/items  # only the name attribute is returned in properties
   curl https://example.org/collections/lakes/items/{item_id}  # only the name attribute is returned in properties
 
+Limiting data responses
+-----------------------
+
+pygeoapi defines a ``limits`` configuration parameter that will allow a user to define default and maximum limits for multiple data types. This parameter is defined at the server level (``server.limits``) with the ability to override at resource level (``resources[*].limits``). An example of this setting is shown below:
+
+.. code-block:: yaml
+
+   limits:
+       default_items: 10  # applies to vector data
+       max_items: 500  # applies to vector data
+       max_distance_x: 123  # applies to all datasets
+       max_distance_y: 456 # applies to all datasets
+       max_distance_units: m  # as per UCUM https://ucum.org/ucum#section-Tables-of-Terminal-Symbols
+       on_exceed: error  # one of error, throttle
+
+The ``limits`` setting is applied as follows:
+
+- can be defined at both the server and resources levels, with resource limits overriding server wide limits settings
+- ``on_exceed`` can be set to ``error`` or ``throttle`` (default).  If a client specified limit exceeds those set by the server:
+  - when set to ``error``, an exception is returned
+  - when set to ``throttle`` the maximum data allowed by the collection/server/provider is returned
+
+Vector data (features, records)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+- when a limit not specified by the client, ``limits.default_items`` can be used to set the result set size
+- when a limit is specified by the client, the minimum of the ``limit`` parameter and ``limits.max_items`` is calculated to set the result set size
+
+Raster data (coverages, environmental data retrieval)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+- when a bbox or spatial subset is specified by the client, ``limits.max_distance_x``, ``limits.max_distance_y`` and ``limits.max_distance_units`` are used to determine whether a request has asked for more data than the collection is configured to provide and respond accordingly (via ``on_exceed``)
 
 Linked Data
 -----------
@@ -638,6 +664,17 @@ deployment flexibility, the path can be specified with string interpolation of e
    The template ``tests/data/base.jsonld`` renders the unmodified JSON-LD. For more information on the capacities
    of Jinja2 templates, see :ref:`html-templating`.
 
+Validating the configuration
+----------------------------
+
+To ensure your configuration is valid, pygeoapi provides a validation
+utility that can be run as follows:
+
+.. code-block:: bash
+
+   pygeoapi config validate -c /path/to/my-pygeoapi-config.yml
+
+
 Summary
 -------
 

docs/source/cql.rst

@@ -3,15 +3,18 @@
 CQL support
 ===========
 
+OGC Common Query Language (`CQL2`_) is a generic language designed to provide enhanced query and subset/filtering to (primarily) feature and record data.
+
 Providers
 ---------
 
-As of now the available providers supported for CQL filtering are limited to :ref:`Elasticsearch <Elasticsearch>` and :ref:`PostgreSQL <PostgreSQL>`.
- 
+CQL2 support is implemented in various pygeoapi feature and record providers.  See the :ref:`feature <ogcapi-features>` and :ref:`metadata <ogcapi-records>` provider sections
+for current provider support.
+
 Limitations
 -----------
 
-Support of CQL is limited to `Simple CQL filter <https://portal.ogc.org/files/96288#cql-core>`_ and thus it allows to query with the
+Support of CQL is limited to `Basic CQL2 <https://docs.ogc.org/is/21-065r2/21-065r2.html#cql2-core>`_ and thus it allows to query with the
 following predicates:
 
 * comparison predicates
@@ -21,20 +24,20 @@ following predicates:
 Formats
 -------
 
-At the moment Elasticsearch supports only the CQL dialect with the JSON encoding `CQL-JSON <https://portal.ogc.org/files/96288#simple-cql-JSON>`_.
+Supported providers leverage the CQL2 dialect with the JSON encoding `CQL-JSON <https://docs.ogc.org/is/21-065r2/21-065r2.html#cql2-json>`_.
 
-PostgreSQL supports both CQL-JSON and CQL-text dialects, `CQL-JSON <https://portal.ogc.org/files/96288#simple-cql-JSON>`_ and `CQL-TEXT <https://portal.ogc.org/files/96288#simple-cql-text>`_
+PostgreSQL supports both `CQL2 JSON <https://docs.ogc.org/is/21-065r2/21-065r2.html#cql2-json>`_ and `CQL text <https://docs.ogc.org/is/21-065r2/21-065r2.html#cql2-text>`_ dialects.
 
 Queries
 ^^^^^^^
 
 The PostgreSQL provider uses `pygeofilter <https://github.com/geopython/pygeofilter>`_ allowing a range of filter expressions, see examples for:
 
-* `Comparison predicates <https://portal.ogc.org/files/96288#simple-cql_comparison-predicates>`_
-* `Spatial predicates <https://portal.ogc.org/files/96288#enhanced-spatial-operators>`_
-* `Temporal predicates <https://portal.ogc.org/files/96288#simple-cql_temporal>`_
+* `Comparison predicates (`Advanced <https://docs.ogc.org/is/21-065r2/21-065r2.html#advanced-comparison-operators>`_, `Case-insensitive <https://docs.ogc.org/is/21-065r2/21-065r2.html#case-insensitive-comparison>`_)
+* `Spatial predicates <https://docs.ogc.org/is/21-065r2/21-065r2.html#spatial-functions>`_
+* `Temporal predicates <https://docs.ogc.org/is/21-065r2/21-065r2.html#temporal-functions>`_
 
-Using Elasticsearch the following type of queries are supported right now:
+Using Elasticsearch the following type of queries are supported currently:
 
 * ``between`` predicate query
 * Logical ``and`` query with ``between`` and ``eq`` expression
@@ -59,11 +62,11 @@ A ``BETWEEN`` example for a specific property through an HTTP POST request:
   curl --location --request POST 'http://localhost:5000/collections/nhsl_hazard_threat_all_indicators_s_bc/items?f=json&limit=50&filter-lang=cql-json' \
   --header 'Content-Type: application/query-cql-json' \
   --data-raw '{
-    "between": {
-      "value": { "property": "properties.MHn_Intensity" },
-      "lower": 0.59,
-      "upper": 0.60
-    }
+    "op": "between",
+    "args": [
+        {"property": "properties.MHn_Intensity"},
+        [0.59, 0.60]
+    ]
   }'
 
 Or 
@@ -73,11 +76,11 @@ Or
   curl --location --request POST 'http://localhost:5000/collections/recentearthquakes/items?f=json&limit=10&filter-lang=cql-json' 
   --header 'Content-Type: application/query-cql-json' 
   --data-raw '{ 
-    "between":{
-      "value":{"property": "ml"},
-      "lower":4,
-      "upper":4.5
-    }
+    "op": "between",
+    "args": [
+        {"property": "ml"},
+        [4, 4.5]
+    ]
   }'
 
 The same ``BETWEEN`` query using HTTP GET request formatted as CQL text and URL encoded as below:
@@ -93,7 +96,11 @@ An ``EQUALS`` example for a specific property:
   curl --location --request POST 'http://localhost:5000/collections/recentearthquakes/items?f=json&limit=10&filter-lang=cql-json' 
   --header 'Content-Type: application/query-cql-json' 
   --data-raw '{
-      "eq":[{"property": "user_entered"},"APBE"]
+    "op": "=",
+    "args": [
+      {"property": "user_entered"},
+      "APBE"
+    ]
   }'
 
 A ``CROSSES`` example via an HTTP GET request.  The CQL text is passed via the ``filter`` parameter.
@@ -115,7 +122,6 @@ The same example, but this time providing a geometry in EWKT format:
 
   curl "http://localhost:5000/collections/beni/items?filter=DWITHIN(geometry,SRID=3857;POINT(1392921%205145517),100,meters)"
 
-
-
-
 Note that the CQL text has been URL encoded. This is required in curl commands but when entering in a browser, plain text can be used e.g. ``CROSSES(foo_geom, LINESTRING(28 -2, 30 -4))``.
+
+.. _`CQL2`: https://docs.ogc.org/is/21-065r2/21-065r2.html

docs/source/data-publishing/ogcapi-features.rst

@@ -31,7 +31,7 @@ parameters.
    `Parquet`_,✅/✅,results/hits,✅,✅,❌,✅,❌,❌,✅
    `PostgreSQL`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅
    `SQLiteGPKG`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,✅
-   `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅
+   `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅
    `Socrata`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅
    `TinyDB`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅