functional recursive nested-processes JSON schema and body deserializ…

…ation

CHANGES.rst

@@ -75,6 +75,7 @@ Fixes:
 - Add the appropriate HTTP error type to respect ``/conf/dru/deploy/unsupported-content-type``
   (fixes `#624 <https://github.com/crim-ca/weaver/issues/624>`_).
 - Fix S3 bucket storage for result file missing the output ID in the path to match local WPS output storage structure.
+- Fix rendering of the ``deprecated`` property in `OpenAPI` representation.
 
 .. _changes_5.9.0:
 

weaver/wps_restapi/colander_extras.py

@@ -53,6 +53,7 @@
 """
 
 # pylint: disable=E0241,duplicate-bases
+# pylint: disable=C0209,consider-using-f-string
 
 import copy
 import inspect
@@ -62,7 +63,6 @@
 from typing import TYPE_CHECKING
 
 import colander
-from beaker.util import deserialize
 from cornice_swagger.converters.exceptions import ConversionError, NoSuchConverter
 from cornice_swagger.converters.parameters import (
     BodyParameterConverter,
@@ -75,8 +75,13 @@
 from cornice_swagger.converters.schema import (
     STRING_FORMATTERS,
     BaseStringTypeConverter,
+    BooleanTypeConverter,
+    DateTimeTypeConverter,
+    DateTypeConverter,
+    IntegerTypeConverter,
     NumberTypeConverter,
     ObjectTypeConverter,
+    TimeTypeConverter,
     TypeConversionDispatcher,
     TypeConverter,
     ValidatorConversionDispatcher,
@@ -120,7 +125,57 @@
 except AttributeError:  # Python 3.6 backport  # pragma: no cover
     RegexPattern = type(re.compile("_"))
 
-# pylint: disable=C0209,consider-using-f-string
+
+class MetadataTypeConverter(TypeConverter):
+    """
+    Converter that applies :term:`OpenAPI` schema metadata properties defined in the schema node.
+    """
+    def convert_type(self, schema_node):
+        result = super(MetadataTypeConverter, self).convert_type(schema_node)
+        deprecated = getattr(schema_node, "deprecated", False)
+        if deprecated:
+            result["deprecated"] = True
+        return result
+
+
+class ExtendedStringTypeConverter(MetadataTypeConverter, BaseStringTypeConverter):
+    pass
+
+
+class ExtendedDateTypeConverter(MetadataTypeConverter, DateTypeConverter):
+    pass
+
+
+class ExtendedTimeTypeConverter(MetadataTypeConverter, TimeTypeConverter):
+    pass
+
+
+class ExtendedDateTimeTypeConverter(MetadataTypeConverter, DateTimeTypeConverter):
+    pass
+
+
+class ExtendedBooleanTypeConverter(MetadataTypeConverter, BooleanTypeConverter):
+    pass
+
+
+class ExtendedIntegerTypeConverter(MetadataTypeConverter, IntegerTypeConverter):
+    pass
+
+
+class ExtendedNumberTypeConverter(MetadataTypeConverter, NumberTypeConverter):
+    pass
+
+
+class ExtendedFloatTypeConverter(ExtendedNumberTypeConverter):
+    format = "float"
+
+
+class ExtendedDecimalTypeConverter(ExtendedNumberTypeConverter):
+    format = "decimal"
+
+
+class ExtendedMoneyTypeConverter(ExtendedDecimalTypeConverter):
+    pass
 
 
 LITERAL_SCHEMA_TYPES = frozenset([
@@ -142,9 +197,12 @@
 URI_REGEX = rf"{URL_REGEX[:-1]}(?:#?|[#?]\S+)$"
 URI = colander.Regex(URI_REGEX, msg=colander._("Must be a URI"), flags=re.IGNORECASE)
 STRING_FORMATTERS.update({
-    "uri": {"converter": BaseStringTypeConverter, "validator": URI},
-    "url": {"converter": BaseStringTypeConverter, "validator": URL},
-    "file": {"converter": BaseStringTypeConverter, "validator": FILE_URI},
+    "uri": {"converter": ExtendedStringTypeConverter, "validator": URI},
+    "url": {"converter": ExtendedStringTypeConverter, "validator": URL},
+    "file": {"converter": ExtendedStringTypeConverter, "validator": FILE_URI},
+    "date": {"converter": ExtendedDateTimeTypeConverter},
+    "time": {"converter": ExtendedDateTimeTypeConverter},
+    "date-time": {"converter": ExtendedDateTimeTypeConverter},
 })
 
 
@@ -1844,6 +1902,7 @@ class KeywordMapper(ExtendedMappingSchema):
     _keyword_map = {_kw: _kw.replace("_of", "Of").replace("_", "") for _kw in _keywords}  # kw->name
     _keyword_inv = {_kn: _kw for _kw, _kn in _keyword_map.items()}                        # name->kw
     _keyword = None  # type: str
+    keywords = frozenset(_keyword_map.values())
 
     def __init__(self, *args, **kwargs):
         super(KeywordMapper, self).__init__(*args, **kwargs)
@@ -2758,7 +2817,7 @@ def convert_type(self, schema_node):
         return converted
 
 
-class DecimalTypeConverter(NumberTypeConverter):
+class DecimalTypeConverter(MetadataTypeConverter, NumberTypeConverter):
     format = "decimal"
 
     def convert_type(self, schema_node):
@@ -2775,11 +2834,11 @@ class MoneyTypeConverter(DecimalTypeConverter):
     )
 
 
-class NoneTypeConverter(TypeConverter):
+class NoneTypeConverter(ExtendedTypeConverter):
     type = "null"
 
 
-class AnyTypeConverter(TypeConverter):
+class AnyTypeConverter(ExtendedTypeConverter):
     def convert_type(self, schema_node):
         converted = super().convert_type(schema_node)
         converted.pop("type", None)
@@ -2810,8 +2869,16 @@ def __init__(self, custom_converters=None, default_converter=None):
         #   user custom converters can override everything, but they must use extended classes to use extra features
         extended_converters = {
             colander.Mapping: VariableObjectTypeConverter,
-            colander.Decimal: DecimalTypeConverter,
-            colander.Money: MoneyTypeConverter,
+            colander.Decimal: ExtendedDecimalTypeConverter,
+            colander.Money: ExtendedMoneyTypeConverter,
+            colander.Float: ExtendedFloatTypeConverter,
+            colander.Number: ExtendedNumberTypeConverter,
+            colander.Integer: ExtendedIntegerTypeConverter,
+            colander.Boolean: ExtendedBooleanTypeConverter,
+            colander.DateTime: ExtendedDateTimeTypeConverter,
+            colander.Date: ExtendedDateTypeConverter,
+            colander.Time: ExtendedTimeTypeConverter,
+            colander.String: ExtendedStringTypeConverter,
             NoneType: NoneTypeConverter,
             AnyType: AnyTypeConverter,
         }
@@ -2981,8 +3048,13 @@ def from_schema(self, schema_node, base_name=None):
         return schema_ret
 
     def _ref_recursive(self, schema, depth, base_name=None):
-        # avoid key error if dealing with 'AnyType'
-        if not schema or not schema.get("type"):
+        # avoid key error if dealing with "any" type
+        # note:
+        #   It is important to consider that keyword mappings will not have a 'type',
+        #   but their child nodes must be iterated to generate '$ref'. We only want to
+        #   avoid the error if the 'type' happens to be explicitly set to an 'any' value, or
+        #   that it is omitted for a generic JSON schema object that does not have a keyword.
+        if not schema or (not schema.get("type") and not any(kw in schema for kw in KeywordMapper.keywords)):
             return schema or {}
         return super()._ref_recursive(schema, depth, base_name=base_name)
 

weaver/wps_restapi/swagger_definitions.py

@@ -114,6 +114,7 @@
     NO_DOUBLE_SLASH_PATTERN,
     AllOfKeywordSchema,
     AnyOfKeywordSchema,
+    AnyType,
     BoundedRange,
     CommaSeparated,
     EmptyMappingSchema,
@@ -127,6 +128,7 @@
     ExtendedString as String,
     NoneType,
     NotKeywordSchema,
+    OAS3DefinitionHandler,
     OneOfCaseInsensitive,
     OneOfKeywordSchema,
     PermissiveMappingSchema,
@@ -3936,6 +3938,7 @@ class ExecuteCollectionInput(FilterSchema, SortBySchema, PermissiveMappingSchema
 
 
 class ExecuteNestedProcessReference(ExtendedMappingSchema):
+    title = "ExecuteNestedProcessReference"
     # 'process' is required for a nested definition, otherwise it will not even be detected as one!
     process = ProcessURL(description="Process reference to be executed.")
 
@@ -3950,12 +3953,18 @@ class ExecuteNestedProcessParameters(ExtendedMappingSchema):
     .. seealso::
         - https://docs.pylonsproject.org/projects/colander/en/latest/binding.html
     """
+    title = "ExecuteNestedProcessParameters"
     _sort_first = ["process", "inputs", "outputs", "properties", "mode", "response"]
+    _schema_extra = {
+        "type": null,
+        "title": "ExecuteNestedProcessParameters",
+        "$ref": f"{OAS3DefinitionHandler.json_pointer}ExecuteProcessParameters"
+    }
 
     @colander.deferred
     def _children(self, __bindings):
         # type: (Dict[str, Any]) -> List[colander.SchemaNode]
-        self.children = [node.clone() for node in ExecuteParameters().children]
+        self.children = [node.clone() for node in ExecuteProcessParameters().children]
         for child in self.children:
             # avoid inserting nested default properties that were omitted (ie: mode/response)
             # they should be included explicitly only on the top-most process by 'Execute(ExecuteParameters)' schema
@@ -3983,6 +3992,7 @@ def deserialize(self, cstruct):
 
 class ExecuteNestedProcessInput(AllOfKeywordSchema):
     _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml"
+    title = "ExecuteNestedProcessInput"
     description = "Nested process to execute, for which the selected output will become the input of the parent call."
 
     _all_of = [
@@ -4002,6 +4012,7 @@ class ExecuteInputAnyType(OneOfKeywordSchema):
     """
     Permissive variants that we attempt to parse automatically.
     """
+    title = "ExecuteInputAnyType"
     _one_of = [
         # Array of literal data with 'data' key
         ArrayLiteralDataType(),
@@ -4023,12 +4034,16 @@ class ExecuteInputAnyType(OneOfKeywordSchema):
     ]
 
 
-class ExecuteInputItem(ExecuteInputDataType, ExecuteInputAnyType):
+class ExecuteInputItem(AllOfKeywordSchema):
     description = (
         "Default value to be looked for uses key 'value' to conform to older drafts of OGC-API standard. "
         "Even older drafts that allowed other fields 'data' instead of 'value' and 'reference' instead of 'href' "
         "are also looked for to remain back-compatible."
     )
+    _all_of = [
+        ExecuteInputDataType(),
+        ExecuteInputAnyType(),
+    ]
 
 
 # backward compatible definition:
@@ -4296,13 +4311,6 @@ class ExecuteParameters(ExecuteInputOutputs):
     These parameters can be either for a top-level process job, or any nested process call.
     """
     _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml"
-    examples = {
-        "ExecuteJSON": {
-            "summary": "Execute a process job using REST JSON payload with OGC API schema.",
-            "value": EXAMPLES["job_execute.json"],
-        },
-    }
-    title = JobTitle(missing=drop)
     mode = JobExecuteModeEnum(
         missing=drop,
         default=ExecuteMode.AUTO,
@@ -4322,22 +4330,32 @@ class ExecuteParameters(ExecuteInputOutputs):
     subscribers = JobExecuteSubscribers(missing=drop)
 
 
-class Execute(ExecuteParameters):
-    """
-    Main execution parameters that can be submitted to run a process.
-
-    Additional parameters are only applicable to the top-most process in a nested definition.
-    """
-    # OGC 'execute.yaml' does not enforce any required item
-    description = "Process execution parameters."
+class ExecuteProcessParameters(ExecuteParameters):
+    title = "ExecuteProcessParameters"
     _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml"
+    _sort_first = [
+        "title",
+        "process",
+        "inputs",
+        "outputs",
+        "properties",
+        "mode",
+        "response",
+        "subscribers",
+    ]
+    _title = JobTitle(name="title", missing=drop)
     process = ProcessURL(
         missing=drop,
         description=(
             "Process reference to be executed. "
             "This parameter is required if the process cannot be inferred from the request endpoint."
         ),
+        example="https://example.com/processes/example"
     )
+
+
+class ExecuteJobParameters(ExtendedMappingSchema):
+    _title = JobTitle(name="title", missing=drop)
     status = JobStatusCreate(
         description=(
             "Status to request creation of the job without submitting it to processing queue "
@@ -4348,6 +4366,38 @@ class Execute(ExecuteParameters):
     )
 
 
+class Execute(AllOfKeywordSchema):
+    """
+    Main execution parameters that can be submitted to run a process.
+
+    Additional parameters are only applicable to the top-most process in a nested definition.
+    """
+    # OGC 'execute.yaml' does not enforce any required item
+    description = "Process execution parameters."
+    examples = {
+        "ExecuteJSON": {
+            "summary": "Execute a process job using REST JSON payload with OGC API schema.",
+            "value": EXAMPLES["job_execute.json"],
+        },
+    }
+    _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml"
+    _sort_first = [
+        "title",
+        "status",
+        "process",
+        "inputs",
+        "outputs",
+        "properties",
+        "mode",
+        "response",
+        "subscribers",
+    ]
+    _all_of = [
+        ExecuteJobParameters(),
+        ExecuteProcessParameters(),
+    ]
+
+
 class QuoteStatusSchema(ExtendedSchemaNode):
     schema_type = String
     validator = OneOf(QuoteStatus.values())