Job output transform

CHANGES.rst

@@ -12,11 +12,17 @@ Changes
 
 Changes:
 --------
-- No change.
+- Add support for various GeoTIFF formats, allowing flexible handling and representation of GeoTIFFs in outputs
+  (fixes `#100 <https://github.com/crim-ca/weaver/issues/100>`_).
+- Add support for ``GET /jobs/{jobId}/results/{id}`` and ``GET /jobs/{jobId}/outputs/{id}`` routes to enable direct access to
+  individual `Job` result items by ID. This enhancement includes support alternate representations based on the ``Accept`` header.
+  If an alternate format (e.g., YAML for a JSON source) is requested it will be automatically generated and returned.
+- Return ``Link`` headers containing all possible output formats, allowing retrieval via query parameters
+  (e.g., ``/jobs/{jobId}/outputs/{id}?f=application/x-yaml``) (fixes `#18 <https://github.com/crim-ca/weaver/issues/18>`_).
 
 Fixes:
 ------
-- No change.
+- Add a fix entry for the multiple duplicate links not re-downloaded in the CLI.
 
 .. _changes_6.4.1:
 

Makefile

@@ -17,6 +17,7 @@ DOCKER_REPO ?= pavics/weaver
 # guess OS (Linux, Darwin,...)
 OS_NAME := $(shell uname -s 2>/dev/null || echo "unknown")
 CPU_ARCH := $(shell uname -m 2>/dev/null || uname -p 2>/dev/null || echo "unknown")
+SUDO ?=
 
 # conda
 CONDA_CMD      ?= __EMPTY__
@@ -229,10 +230,10 @@ conda-env-export:		## export the conda environment
 install: install-all    ## alias for 'install-all' target
 
 .PHONY: install-run
-install-run: conda-install install-sys install-pkg install-raw 	## install requirements and application to run locally
+install-run: conda-install install-sys install-pkg install-raw install-dev install-transform ## install requirements and application to run locally
 
 .PHONY: install-all
-install-all: conda-install install-sys install-pkg install-pip install-dev  ## install application with all dependencies
+install-all: conda-install install-sys install-pkg install-pip install-dev install-transform ## install application with all dependencies
 
 .PHONY: install-doc
 install-doc: install-pip	## install documentation dependencies
@@ -275,7 +276,7 @@ install-raw:	## install without any requirements or dependencies (suppose everyt
 install-npm:	## install npm package manager and dependencies if they cannot be found
 	@[ -f "$(shell which npm)" ] || ( \
 		echo "Binary package manager npm not found. Attempting to install it."; \
-		apt-get install npm \
+		$(SUDO) apt-get install npm \
 	)
 
 .PHONY: install-npm-stylelint
@@ -292,6 +293,16 @@ install-npm-remarklint: install-npm		## install remark-lint dependency for 'chec
 		npm install --save-dev \
 	)
 
+.PHONY: install-transform
+install-transform: install-cairo-dependencies       # install-transform dependencies
+
+.PHONY: install-cairo-dependencies
+install-cairo-dependencies:   ## install required dependencies for Transformer
+	@[ -f "$(shell which cairo)" ] || ( \
+		echo "Binary package manager cairo not found. Attempting to install it."; \
+		$(SUDO) apt-get install libpangocairo-1.0-0 \
+	)
+
 .PHONY: install-dev-npm
 install-dev-npm: install-npm install-npm-remarklint install-npm-remarklint  ## install all npm development dependencies
 

docker/Dockerfile-base

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         g++ \
         git \
         nodejs \
+        libpangocairo-1.0-0 \
     && pip install --no-cache-dir --upgrade -r requirements-sys.txt \
     && pip install --no-cache-dir -r requirements.txt \
     && pip install --no-cache-dir -e ${APP_DIR} \

requirements.txt

@@ -11,6 +11,7 @@ boto3-stubs[s3]
 # https://github.com/celery/billiard/issues/313
 billiard>2; sys_platform != "win32"  # avoid issue with use_2to3
 billiard>3.2,<3.4; sys_platform == "win32"
+cairosvg
 # pymongo>=4 breaks for some kombu combinations corresponding to pinned Celery
 # - https://github.com/crim-ca/weaver/issues/386
 # - https://github.com/celery/kombu/pull/1536
@@ -52,6 +53,7 @@ duration
 esgf-compute-api @ git+https://github.com/ESGF/[email protected]
 # invalid 'zarr' requirement in 'geotiff' dependencies required by 'pywps' fail to install
 # (https://github.com/KipCrossing/geotiff/pull/59)
+fpdf
 geotiff>=0.2.8
 # gunicorn >20 breaks some config.ini loading parameters (paste)
 # use pserve to continue supporting config.ini with paste settings
@@ -60,6 +62,7 @@ gunicorn>=22
 # even more reduced dependency constraints (https://github.com/vinitkumar/json2xml/pull/195)
 json2xml==4.1.0
 jsonschema>=3.0.1
+
 # FIXME: kombu for pymongo>=4 not yet released as 5.3.0 (only pre-releases available)
 # - https://github.com/crim-ca/weaver/issues/386
 # - https://github.com/celery/kombu/pull/1536
@@ -70,12 +73,15 @@ mako
 # force use of later mistune (https://github.com/common-workflow-language/schema_salad/pull/619#issuecomment-1346025607)
 # employed by cwltool -> schema-salad -> mistune
 #mistune>=2.0.3,<2.1
+multipagetiff
 mypy_boto3_s3
 numpy>=1.22.2
 # esgf-compute-api (cwt) needs oauthlib but doesn't add it in their requirements
 oauthlib
 owslib==0.32.1
 PasteDeploy>=3.1.0; python_version >= "3.12"
+pandas
+Pillow
 pint
 psutil
 # notes: https://github.com/geopython/pygeofilter
@@ -103,9 +109,11 @@ pystac
 pystac_client
 python-box
 python-dateutil
+python-magic
 pytz
 pywps==4.6.0
 pyyaml>=5.2
+rasterio
 rdflib>=5  # pyup: ignore
 requests>=2.32.2
 requests_file

tests/functional/test_cli.py

@@ -475,7 +475,11 @@ def test_describe(self):
         for out_fmt in output_formats:
             out_fmt.pop("$schema", None)
             out_fmt.pop("$id", None)
-        assert output_formats == [{"default": True, "mediaType": ContentType.TEXT_PLAIN}]
+        assert output_formats == [
+            {"default": True, "mediaType": ContentType.TEXT_PLAIN},
+            {"mediaType": ContentType.TEXT_HTML},
+            {"mediaType": ContentType.APP_PDF}
+        ]
         assert "undefined" not in result.message, "CLI should not have confused process description as response detail."
         assert result.body["description"] == (
             "Dummy process that simply echo's back the input message for testing purposes."
@@ -1493,6 +1497,13 @@ def test_deploy_payload_process_info_merged(self):
             out_cwl_fmt = {"default": False, "mediaType": io_fmt}
             out_oas_fmt = {"default": True, "mediaType": ContentType.APP_JSON}
             out_any_fmt = [out_cwl_fmt, out_oas_fmt]
+            # Alternative format added in process description
+            out_alt_fmt = [
+                {"mediaType": ContentType.TEXT_CSV},
+                {"mediaType": ContentType.APP_XML},
+                {"mediaType": ContentType.APP_YAML},
+            ]
+            out_any_fmt.extend(out_alt_fmt)
             # ignore schema specifications for comparison only of contents
             for field in ["$id", "$schema"]:
                 in_schema.pop(field, None)