diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 496f1b9..3ee83fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: - name: Install python dependencies run: | - python -m pip install pypgstac==${{ env.PGSTAC_VERSION }} psycopg[pool] httpx pytest pytest-benchmark + python -m pip install pypgstac==${{ env.PGSTAC_VERSION }} psycopg[pool] httpx pytest pytest-benchmark rasterio - name: Ingest Stac Items/Collection run: | diff --git a/CHANGES.md b/CHANGES.md index 0f6888f..112499e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,8 +1,39 @@ # Release Notes -## 0,5,2 (TDB) - -* add `tilejson` URL links for `defaults layers` defined in mosaic's metadata in `/mosaic/register` and `/mosaic/{mosaic_id}/info` response +## 0.6.0 (TDB) + +* add `tilejson` URL links for `layers` defined in mosaic's metadata in `/mosaic/register` and `/mosaic/{mosaic_id}/info` response +* support multiple `layers` in `/mosaic/{mosaic_id}/WMTSCapabilities.xml` endpoint created from mosaic's metadata + +**breaking change** + +* In `/mosaic/WMTSCapabilities.xml` we removed the query-parameters related to the `tile` endpoint (which are forwarded) so `?assets=` is no more required. +The endpoint will still raise an error if there are no `layers` in the mosaic metadata and no required tile's parameters are passed. + + ```python + # before + response = httpx.get("/mosaic/{mosaic_id}/WMTSCapabilities.xml") + assert response.status_code == 400 + + response = httpx.get("/mosaic/{mosaic_id}/WMTSCapabilities.xml?assets=cog") + assert response.status_code == 200 + + # now + # If the mosaic has `defaults` layers set in the metadata + # we will construct a WMTS document with multiple layers, so no need for the user to pass any `assets=` + response = httpx.get("/mosaic/{mosaic_id}/WMTSCapabilities.xml") + assert response.status_code == 200 + with rasterio.open(io.BytesIO(response.content)) as src: + assert src.profile["driver"] == "WMTS" + assert len(src.subdatasets) == 2 + + # If the user pass any valid `tile` parameters, an additional layer will be added to the one from the metadata + response = httpx.get("/mosaic/{mosaic_id}/WMTSCapabilities.xml?assets=cog") + assert response.status_code == 200 + with rasterio.open(io.BytesIO(response.content)) as src: + assert src.profile["driver"] == "WMTS" + assert len(src.subdatasets) == 3 + ``` ## 0.5.1 (2023-08-03) diff --git a/docs/src/intro.md b/docs/src/intro.md index 90219f1..63792a7 100644 --- a/docs/src/intro.md +++ b/docs/src/intro.md @@ -10,7 +10,9 @@ By default the main application (`titiler.pgstac.main.app`) provides two sets of ## Mosaic -#### 1. Register a `Search` request (Mosaic) +The goal of the `mosaic` endpoints is to use any [`search`](https://github.com/radiantearth/stac-api-spec/tree/master/item-search) query to create tiles. `titiler-pgstac` provides a set of endpoint to `register` and `list` the `search` queries. + +### Register a `Search` request ![](https://user-images.githubusercontent.com/10407788/132193537-0560016f-09bc-4a25-8a2a-eac9b50bc28a.png) @@ -70,16 +72,16 @@ curl -X 'POST' 'http://127.0.0.1:8081/mosaic/register' \ } ``` -##### 1.1 Get Mosaic metadata +##### Mosaic metadata ```bash curl http://127.0.0.1:8081/mosaic/5063721f06957d6b2320326d82e90d1e/info | jq >> { "search": { - "hash": "5063721f06957d6b2320326d82e90d1e", - "search": { - "filter": { + "hash": "5063721f06957d6b2320326d82e90d1e", # <-- this is the search/mosaic ID + "search": { # <-- Summary of the search request + "filter": { # <-- this is CQL2 filter associated with the search "op": "and", "args": [ { @@ -129,12 +131,12 @@ curl http://127.0.0.1:8081/mosaic/5063721f06957d6b2320326d82e90d1e/info | jq ] } }, - "_where": "( ( (collection_id = 'landsat-c2l2-sr') and st_intersects(geometry, '0103000020E610000001000000050000000000000000F05EC055F6687D502741400000000000F05EC02D553EA94A6943400000000000885DC02D553EA94A6943400000000000885DC055F6687D502741400000000000F05EC055F6687D50274140'::geometry) ) ) ", + "_where": "( ( (collection_id = 'landsat-c2l2-sr') and st_intersects(geometry, '0103000020E610000001000000050000000000000000F05EC055F6687D502741400000000000F05EC02D553EA94A6943400000000000885DC02D553EA94A6943400000000000885DC055F6687D502741400000000000F05EC055F6687D50274140'::geometry) ) ) ", # <-- internal pgstac WHERE expression "orderby": "datetime DESC, id DESC", - "lastused": "2022-03-03T11:44:55.878504+00:00", - "usecount": 2, - "metadata": { - "type": "mosaic" + "lastused": "2022-03-03T11:44:55.878504+00:00", # <-- internal pgstac variable + "usecount": 2, # <-- internal pgstac variable + "metadata": { # <-- titiler-pgstac Mosaic Metadata + "type": "mosaic" # <-- where using the `/mosaic/register` endpoint, titiler-pgstac will add `type=mosaic` to the metadata } }, "links": [ @@ -199,7 +201,7 @@ curl http://127.0.0.1:8081/mosaic/f31d7de8a5ddfa3a80b9a9dd06378db1/info | jq '.s } ``` -#### 2. Fetch mosaic `Tiles` +### Fetch mosaic `Tiles` When we have a `searchid` we can now call the dynamic tiler and ask for Map Tiles. diff --git a/docs/src/mosaic_endpoints.md b/docs/src/mosaic_endpoints.md index 714c505..0e511e7 100644 --- a/docs/src/mosaic_endpoints.md +++ b/docs/src/mosaic_endpoints.md @@ -16,11 +16,12 @@ The `titiler.pgstac` package comes with a full FastAPI application with Mosaic a `:endpoint:/mosaic/register - [POST]` -- **Body**: A valid STAC Search query (see: https://github.com/radiantearth/stac-api-spec/tree/master/item-search) +- **Body** (a combination of Search+Metadata): A JSON body composed of a valid **STAC Search** query (see: https://github.com/radiantearth/stac-api-spec/tree/master/item-search) and Mosaic's metadata. ```json // titiler-pgstac search body example { + // STAC search query "collections": [ "string" ], @@ -28,16 +29,16 @@ The `titiler.pgstac` package comes with a full FastAPI application with Mosaic a "string" ], "bbox": [ - "string", - "string", - "string", - "string" + "number", + "number", + "number", + "number" ], "intersects": { "type": "Point", "coordinates": [ - "string", - "string" + "number", + "number" ] }, "query": { @@ -49,19 +50,21 @@ The `titiler.pgstac` package comes with a full FastAPI application with Mosaic a "datetime": "string", "sortby": "string", "filter-lang": "cql-json", + // titiler-pgstac mosaic metadata "metadata": { "type": "mosaic", "bounds": [ - "string", - "string", - "string", - "string" + "number", + "number", + "number", + "number" ], - "minzoom": 0, - "maxzoom": 0, + "minzoom": "number", + "maxzoom": "number", "name": "string", "assets": [ - "string" + "string", + "string", ], "defaults": {} } @@ -271,32 +274,14 @@ Example: - **TileMatrixSetId**: TileMatrixSet name, default is `WebMercatorQuad`. OPTIONAL - QueryParams: - - **tile_format**: Output image format, default is set to None and will be either JPEG or PNG depending on masked value. + - **tile_format**: Output image format, default is set to PNG. - **tile_scale**: Tile size scale, default is set to 1 (256x256). OPTIONAL - **minzoom**: Overwrite default minzoom. OPTIONAL - **maxzoom**: Overwrite default maxzoom. OPTIONAL - - **expression** (str): rio-tiler's math expression with asset names (e.g `Asset1_b1/Asset2_b1`). - - **asset_as_band** (bool): tell rio-tiler that each asset is a 1 band dataset, so expression `Asset1/Asset2` can be passed. - - **asset_bidx** (array[str]): Per asset band math expression (e.g `Asset1|1;2;3`). - - **nodata** (str, int, float): Overwrite internal Nodata value. - - **unscale** (bool): Apply dataset internal Scale/Offset. - - **resampling** (str): rasterio resampling method. Default is `nearest`. - - **algorithm** (str): Custom algorithm name (e.g `hillshade`). - - **algorithm_params** (str): JSON encoded algorithm parameters. - - **rescale** (array[str]): Comma (',') delimited Min,Max range (e.g `rescale=0,1000`, `rescale=0,1000&rescale=0,3000&rescale=0,2000`). - - **color_formula** (str): rio-color formula. - - **colormap** (str): JSON encoded custom Colormap. - - **colormap_name** (str): rio-tiler color map name. - - **return_mask** (bool): Add mask to the output data. Default is True. - - **buffer** (float): Add buffer on each side of the tile (e.g 0.5 = 257x257, 1.0 = 258x258). - - **scan_limit** (int): Return as soon as we scan N items, Default is 10,000 in PgSTAC. - - **items_limit** (int): Return as soon as we have N items per geometry, Default is 100 in PgSTAC. - - **time_limit** (int): Return after N seconds to avoid long requests, Default is 5sec in PgSTAC. - - **exitwhenfull** (bool): Return as soon as the geometry is fully covered, Default is `True` in PgSTAC. - - **skipcovered** (bool): Skip any items that would show up completely under the previous items, Default is `True` in PgSTAC. + !!! important - **assets** OR **expression** is required + additional query-parameters will be forwarded to the `tile` URL. If no `defaults` mosaic metadata, **assets** OR **expression** will be required Example: diff --git a/pyproject.toml b/pyproject.toml index 2ac203d..6b1edbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,3 +122,8 @@ no_implicit_optional = true strict_optional = true namespace_packages = true explicit_package_bases = true + +[tool.pytest.ini_options] +filterwarnings = [ + "ignore::rasterio.errors.NotGeoreferencedWarning", +] diff --git a/tests/test_mosaic.py b/tests/test_mosaic.py index 2570910..123a322 100644 --- a/tests/test_mosaic.py +++ b/tests/test_mosaic.py @@ -4,6 +4,7 @@ from datetime import datetime from unittest.mock import patch +import pytest import rasterio from rasterio.crs import CRS @@ -487,9 +488,9 @@ def test_query_with_metadata(app): assert resp["searchid"] assert resp["links"] - cql2_id = resp["searchid"] + mosaic_id = resp["searchid"] - response = app.get(f"/mosaic/{cql2_id}/info") + response = app.get(f"/mosaic/{mosaic_id}/info") assert response.status_code == 200 resp = response.json() assert resp["search"] @@ -508,7 +509,7 @@ def test_query_with_metadata(app): "maxzoom": 2, } - response = app.get(f"/mosaic/{cql2_id}/tilejson.json?assets=cog") + response = app.get(f"/mosaic/{mosaic_id}/tilejson.json?assets=cog") assert response.status_code == 200 resp = response.json() assert resp["minzoom"] == 1 @@ -527,17 +528,22 @@ def test_query_with_metadata(app): "defaults": { "one_band": { "assets": "cog", - "asset_bidx": 1, + "asset_bidx": "cog|1", }, "three_bands": { "assets": "cog", - "asset_bidx": [1, 2, 3], + "asset_bidx": "cog|1,2,3", + }, + # missing `assets` + "bad_layer": { + "asset_bidx": "cog|1,2,3", }, }, }, } - response = app.post("/mosaic/register", json=query) + with pytest.warns(UserWarning): + response = app.post("/mosaic/register", json=query) assert response.status_code == 200 resp = response.json() assert resp["searchid"] @@ -545,10 +551,63 @@ def test_query_with_metadata(app): len(resp["links"]) == 6 ) # info, tilejson, map, wmts tilejson for one_band, tilejson for three_bands link = resp["links"][-2] + + mosaic_id_metadata = resp["searchid"] + assert link["title"] == "TileJSON link for `one_band` layer." - assert "asset_bidx=1" in link["href"] + assert "asset_bidx=cog%7C1" in link["href"] assert "assets=cog" in link["href"] + # Test WMTS + # 1. missing assets and no metadata layers + response = app.get(f"/mosaic/{mosaic_id}/WMTSCapabilities.xml") + assert response.status_code == 400 + assert ( + response.json()["detail"] + == "assets must be defined either via expression or assets options." + ) + + # 2. assets and no metadata layers + response = app.get( + f"/mosaic/{mosaic_id}/WMTSCapabilities.xml", params={"assets": "cog"} + ) + assert response.status_code == 200 + assert response.headers["content-type"] == "application/xml" + + with rasterio.open(io.BytesIO(response.content)) as src: + assert src.crs == "epsg:3857" + assert src.profile["driver"] == "WMTS" + assert not src.subdatasets + + # 3. no assets and metadata layers + with pytest.warns(UserWarning): + response = app.get(f"/mosaic/{mosaic_id_metadata}/WMTSCapabilities.xml") + assert response.status_code == 200 + + assert response.headers["content-type"] == "application/xml" + + with rasterio.open(io.BytesIO(response.content)) as src: + assert src.profile["driver"] == "WMTS" + assert len(src.subdatasets) == 2 + assert src.subdatasets[0].endswith(",layer=one_band") + assert src.subdatasets[1].endswith(",layer=three_bands") + + # 4. assets and metadata layers + with pytest.warns(UserWarning): + response = app.get( + f"/mosaic/{mosaic_id_metadata}/WMTSCapabilities.xml", + params={"assets": "cog"}, + ) + assert response.status_code == 200 + assert response.headers["content-type"] == "application/xml" + + with rasterio.open(io.BytesIO(response.content)) as src: + assert src.profile["driver"] == "WMTS" + assert len(src.subdatasets) == 3 + assert src.subdatasets[0].endswith(",layer=one_band") + assert src.subdatasets[1].endswith(",layer=three_bands") + assert src.subdatasets[2].endswith(",layer=default") + @patch("rio_tiler.io.rasterio.rasterio") def test_statistics(rio, app): diff --git a/titiler/pgstac/factory.py b/titiler/pgstac/factory.py index ca3014d..9e71621 100644 --- a/titiler/pgstac/factory.py +++ b/titiler/pgstac/factory.py @@ -1,8 +1,8 @@ """Custom MosaicTiler Factory for PgSTAC Mosaic Backend.""" - import os import re import sys +import warnings from dataclasses import dataclass from typing import ( Any, @@ -18,10 +18,12 @@ ) from urllib.parse import urlencode +import jinja2 import rasterio from cogeo_mosaic.backends import BaseBackend from cogeo_mosaic.errors import MosaicNotFoundError from fastapi import Body, Depends, HTTPException, Path, Query +from fastapi.dependencies.utils import get_dependant, request_params_to_args from geojson_pydantic import Feature, FeatureCollection from psycopg import sql from psycopg.rows import class_row @@ -32,6 +34,7 @@ from starlette.requests import Request from starlette.responses import HTMLResponse, Response from starlette.routing import NoMatchFound +from starlette.templating import Jinja2Templates from titiler.core.dependencies import ( AssetsBidxExprParams, @@ -67,6 +70,17 @@ def _first_value(values: List[Any], default: Any = None): return next(filter(lambda x: x is not None, values), default) +DEFAULT_TEMPLATES = Jinja2Templates( + directory="", + loader=jinja2.ChoiceLoader( + [ + jinja2.PackageLoader(__package__, "templates"), + jinja2.PackageLoader("titiler.core", "templates"), + ] + ), +) # type:ignore + + @dataclass class MosaicTilerFactory(BaseTilerFactory): """Custom MosaicTiler for PgSTAC Mosaic Backend.""" @@ -95,6 +109,29 @@ class MosaicTilerFactory(BaseTilerFactory): add_mosaic_list: bool = False + templates: Jinja2Templates = DEFAULT_TEMPLATES + + def check_query_params( + self, *, dependencies: List[Callable], query_params: QueryParams + ) -> None: + """Check QueryParams for Query dependency. + + 1. `get_dependant` is used to get the query-parameters required by the `callable` + 2. we use `request_params_to_args` to construct arguments needed to call the `callable` + 3. we call the `callable` and catch any errors + + Important: We assume the `callable` in not a co-routine + + """ + for dependency in dependencies: + dep = get_dependant(path="", call=dependency) + if dep.query_params: + # call the dependency with the query-parameters values + query_values, _ = request_params_to_args(dep.query_params, query_params) + _ = dependency(**query_values) + + return + def register_routes(self) -> None: """This Method register routes to the router.""" self._search_routes() @@ -470,7 +507,6 @@ def wmts( Literal[tuple(self.supported_tms.list())], f"Identifier selecting one of the TileMatrixSetId supported (default: '{self.default_tms}')", ] = self.default_tms, - src_path=Depends(self.path_dependency), tile_format: Annotated[ ImageType, Query(description="Output image type. Default is png."), @@ -489,32 +525,6 @@ def wmts( Optional[int], Query(description="Overwrite default maxzoom."), ] = None, - layer_params=Depends(self.layer_dependency), - dataset_params=Depends(self.dataset_dependency), - pixel_selection=Depends(self.pixel_selection_dependency), - buffer: Annotated[ - Optional[float], - Query( - gt=0, - title="Tile buffer.", - description="Buffer on each side of the given tile. It must be a multiple of `0.5`. Output **tilesize** will be expanded to `tilesize + 2 * buffer` (e.g 0.5 = 257x257, 1.0 = 258x258).", - ), - ] = None, - post_process=Depends(self.process_dependency), - rescale=Depends(self.rescale_dependency), - color_formula: Annotated[ - Optional[str], - Query( - title="Color Formula", - description="rio-color formula (info: https://github.com/mapbox/rio-color)", - ), - ] = None, - colormap=Depends(self.colormap_dependency), - render_params=Depends(self.render_dependency), - pgstac_params: PgSTACParams = Depends(), - backend_params=Depends(self.backend_dependency), - reader_params=Depends(self.reader_dependency), - env=Depends(self.environment_dependency), ): """OGC WMTS endpoint.""" with request.app.state.dbpool.connection() as conn: @@ -536,7 +546,47 @@ def wmts( "format": tile_format.value, "tileMatrixSetId": tileMatrixSetId, } - tiles_url = self.url_for(request, "tile", **route_params) + + # `route_params.copy()` this can be removed after titiler>=0.13.2 update + tiles_url = self.url_for(request, "tile", **route_params.copy()) + + # List of dependencies a `/tile` URL should validate + # Note: Those dependencies should only require Query() inputs + tile_dependencies = [ + self.layer_dependency, + self.dataset_dependency, + self.pixel_selection_dependency, + self.process_dependency, + self.rescale_dependency, + self.colormap_dependency, + self.render_dependency, + PgSTACParams, + self.reader_dependency, + self.backend_dependency, + ] + + layers: List[Dict[str, Any]] = [] + if search_info.metadata.defaults: + for name, values in search_info.metadata.defaults.items(): + query_string = urlencode(values, doseq=True) + try: + self.check_query_params( + dependencies=tile_dependencies, + query_params=QueryParams(query_string), + ) + except Exception as e: + warnings.warn( + f"Cannot construct URL for layer `{name}`: {repr(e)}", + UserWarning, + ) + continue + + layers.append( + { + "name": name, + "endpoint": tiles_url + f"?{query_string}", + } + ) qs_key_to_remove = [ "tilematrixsetid", @@ -555,6 +605,20 @@ def wmts( if qs: tiles_url += f"?{urlencode(qs)}" + # Checking if we can construct a valid tile URL + # 1. we use `check_query_params` to validate the query-parameter + # 2. if there is no layers (from mosaic metadata) we raise the caught error + # 3. if there no errors we then add a default `layer` to the layers stack + try: + self.check_query_params( + dependencies=tile_dependencies, query_params=QueryParams(qs) + ) + except Exception as e: + if not layers: + raise e + else: + layers.append({"name": "default", "endpoint": tiles_url}) + tms = self.supported_tms.get(tileMatrixSetId) minzoom = _first_value([minzoom, search_info.metadata.minzoom], tms.minzoom) maxzoom = _first_value([maxzoom, search_info.metadata.maxzoom], tms.maxzoom) @@ -582,12 +646,11 @@ def wmts( "wmts.xml", { "request": request, - "tiles_endpoint": tiles_url, + "title": search_info.metadata.name or searchid, "bounds": bounds, "tileMatrix": tileMatrix, "tms": tms, - "title": "Mosaic", - "layer_name": "mosaic", + "layers": layers, "media_type": tile_format.mediatype, }, media_type=MediaType.xml.value, @@ -717,7 +780,35 @@ def register_search( pass if search_info.metadata.defaults: + # List of dependencies a `/tile` URL should validate + # Note: Those dependencies should only require Query() inputs + tile_dependencies = [ + self.layer_dependency, + self.dataset_dependency, + self.pixel_selection_dependency, + self.process_dependency, + self.rescale_dependency, + self.colormap_dependency, + self.render_dependency, + PgSTACParams, + self.reader_dependency, + self.backend_dependency, + ] + for name, values in search_info.metadata.defaults.items(): + query_string = urlencode(values, doseq=True) + try: + self.check_query_params( + dependencies=tile_dependencies, + query_params=QueryParams(query_string), + ) + except Exception as e: + warnings.warn( + f"Cannot construct URL for layer `{name}`: {repr(e)}", + UserWarning, + ) + continue + links.append( model.Link( title=f"TileJSON link for `{name}` layer.", @@ -727,7 +818,7 @@ def register_search( "tilejson", searchid=search_info.id, ) - + f"?{urlencode(values, doseq=True)}", + + f"?{query_string}", ) ) diff --git a/titiler/pgstac/templates/wmts.xml b/titiler/pgstac/templates/wmts.xml new file mode 100644 index 0000000..ea8e6c1 --- /dev/null +++ b/titiler/pgstac/templates/wmts.xml @@ -0,0 +1,64 @@ + + + "{{ title }}" + OGC WMTS + 1.0.0 + + + + + + + + + RESTful + + + + + + + + + + + + + RESTful + + + + + + + + + {% for layer in layers %} + + {{ title }} + {{ layer.name }} + {{ layer.name }} + + {{ bounds[0] }} {{ bounds[1] }} + {{ bounds[2] }} {{ bounds[3] }} + + + {{ media_type }} + + {{ tms.id }} + + + + {% endfor %} + + {{ tms.id }} + {{ tms.crs.srs }} + {% for item in tileMatrix %} + {{ item | safe }} + {% endfor %} + + + +