From 8025528bf4cb3dcc616b6fdcd4e522edf552ed58 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 6 Nov 2024 04:06:32 -0500 Subject: [PATCH] functional recursive nested-processes JSON schema and body deserialization --- CHANGES.rst | 1 + weaver/wps_restapi/colander_extras.py | 96 ++++++++++++++++++++--- weaver/wps_restapi/swagger_definitions.py | 84 ++++++++++++++++---- 3 files changed, 152 insertions(+), 29 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index eeae25142..00f64e362 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -75,6 +75,7 @@ Fixes: - Add the appropriate HTTP error type to respect ``/conf/dru/deploy/unsupported-content-type`` (fixes `#624 `_). - Fix S3 bucket storage for result file missing the output ID in the path to match local WPS output storage structure. +- Fix rendering of the ``deprecated`` property in `OpenAPI` representation. .. _changes_5.9.0: diff --git a/weaver/wps_restapi/colander_extras.py b/weaver/wps_restapi/colander_extras.py index 29597f46e..8ea59e907 100644 --- a/weaver/wps_restapi/colander_extras.py +++ b/weaver/wps_restapi/colander_extras.py @@ -53,6 +53,7 @@ """ # pylint: disable=E0241,duplicate-bases +# pylint: disable=C0209,consider-using-f-string import copy import inspect @@ -62,7 +63,6 @@ from typing import TYPE_CHECKING import colander -from beaker.util import deserialize from cornice_swagger.converters.exceptions import ConversionError, NoSuchConverter from cornice_swagger.converters.parameters import ( BodyParameterConverter, @@ -75,8 +75,13 @@ from cornice_swagger.converters.schema import ( STRING_FORMATTERS, BaseStringTypeConverter, + BooleanTypeConverter, + DateTimeTypeConverter, + DateTypeConverter, + IntegerTypeConverter, NumberTypeConverter, ObjectTypeConverter, + TimeTypeConverter, TypeConversionDispatcher, TypeConverter, ValidatorConversionDispatcher, @@ -120,7 +125,57 @@ except AttributeError: # Python 3.6 backport # pragma: no cover RegexPattern = type(re.compile("_")) -# pylint: disable=C0209,consider-using-f-string + +class MetadataTypeConverter(TypeConverter): + """ + Converter that applies :term:`OpenAPI` schema metadata properties defined in the schema node. + """ + def convert_type(self, schema_node): + result = super(MetadataTypeConverter, self).convert_type(schema_node) + deprecated = getattr(schema_node, "deprecated", False) + if deprecated: + result["deprecated"] = True + return result + + +class ExtendedStringTypeConverter(MetadataTypeConverter, BaseStringTypeConverter): + pass + + +class ExtendedDateTypeConverter(MetadataTypeConverter, DateTypeConverter): + pass + + +class ExtendedTimeTypeConverter(MetadataTypeConverter, TimeTypeConverter): + pass + + +class ExtendedDateTimeTypeConverter(MetadataTypeConverter, DateTimeTypeConverter): + pass + + +class ExtendedBooleanTypeConverter(MetadataTypeConverter, BooleanTypeConverter): + pass + + +class ExtendedIntegerTypeConverter(MetadataTypeConverter, IntegerTypeConverter): + pass + + +class ExtendedNumberTypeConverter(MetadataTypeConverter, NumberTypeConverter): + pass + + +class ExtendedFloatTypeConverter(ExtendedNumberTypeConverter): + format = "float" + + +class ExtendedDecimalTypeConverter(ExtendedNumberTypeConverter): + format = "decimal" + + +class ExtendedMoneyTypeConverter(ExtendedDecimalTypeConverter): + pass LITERAL_SCHEMA_TYPES = frozenset([ @@ -142,9 +197,12 @@ URI_REGEX = rf"{URL_REGEX[:-1]}(?:#?|[#?]\S+)$" URI = colander.Regex(URI_REGEX, msg=colander._("Must be a URI"), flags=re.IGNORECASE) STRING_FORMATTERS.update({ - "uri": {"converter": BaseStringTypeConverter, "validator": URI}, - "url": {"converter": BaseStringTypeConverter, "validator": URL}, - "file": {"converter": BaseStringTypeConverter, "validator": FILE_URI}, + "uri": {"converter": ExtendedStringTypeConverter, "validator": URI}, + "url": {"converter": ExtendedStringTypeConverter, "validator": URL}, + "file": {"converter": ExtendedStringTypeConverter, "validator": FILE_URI}, + "date": {"converter": ExtendedDateTimeTypeConverter}, + "time": {"converter": ExtendedDateTimeTypeConverter}, + "date-time": {"converter": ExtendedDateTimeTypeConverter}, }) @@ -1844,6 +1902,7 @@ class KeywordMapper(ExtendedMappingSchema): _keyword_map = {_kw: _kw.replace("_of", "Of").replace("_", "") for _kw in _keywords} # kw->name _keyword_inv = {_kn: _kw for _kw, _kn in _keyword_map.items()} # name->kw _keyword = None # type: str + keywords = frozenset(_keyword_map.values()) def __init__(self, *args, **kwargs): super(KeywordMapper, self).__init__(*args, **kwargs) @@ -2758,7 +2817,7 @@ def convert_type(self, schema_node): return converted -class DecimalTypeConverter(NumberTypeConverter): +class DecimalTypeConverter(MetadataTypeConverter, NumberTypeConverter): format = "decimal" def convert_type(self, schema_node): @@ -2775,11 +2834,11 @@ class MoneyTypeConverter(DecimalTypeConverter): ) -class NoneTypeConverter(TypeConverter): +class NoneTypeConverter(ExtendedTypeConverter): type = "null" -class AnyTypeConverter(TypeConverter): +class AnyTypeConverter(ExtendedTypeConverter): def convert_type(self, schema_node): converted = super().convert_type(schema_node) converted.pop("type", None) @@ -2810,8 +2869,16 @@ def __init__(self, custom_converters=None, default_converter=None): # user custom converters can override everything, but they must use extended classes to use extra features extended_converters = { colander.Mapping: VariableObjectTypeConverter, - colander.Decimal: DecimalTypeConverter, - colander.Money: MoneyTypeConverter, + colander.Decimal: ExtendedDecimalTypeConverter, + colander.Money: ExtendedMoneyTypeConverter, + colander.Float: ExtendedFloatTypeConverter, + colander.Number: ExtendedNumberTypeConverter, + colander.Integer: ExtendedIntegerTypeConverter, + colander.Boolean: ExtendedBooleanTypeConverter, + colander.DateTime: ExtendedDateTimeTypeConverter, + colander.Date: ExtendedDateTypeConverter, + colander.Time: ExtendedTimeTypeConverter, + colander.String: ExtendedStringTypeConverter, NoneType: NoneTypeConverter, AnyType: AnyTypeConverter, } @@ -2981,8 +3048,13 @@ def from_schema(self, schema_node, base_name=None): return schema_ret def _ref_recursive(self, schema, depth, base_name=None): - # avoid key error if dealing with 'AnyType' - if not schema or not schema.get("type"): + # avoid key error if dealing with "any" type + # note: + # It is important to consider that keyword mappings will not have a 'type', + # but their child nodes must be iterated to generate '$ref'. We only want to + # avoid the error if the 'type' happens to be explicitly set to an 'any' value, or + # that it is omitted for a generic JSON schema object that does not have a keyword. + if not schema or (not schema.get("type") and not any(kw in schema for kw in KeywordMapper.keywords)): return schema or {} return super()._ref_recursive(schema, depth, base_name=base_name) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 39f55e951..0ca276c7a 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -114,6 +114,7 @@ NO_DOUBLE_SLASH_PATTERN, AllOfKeywordSchema, AnyOfKeywordSchema, + AnyType, BoundedRange, CommaSeparated, EmptyMappingSchema, @@ -127,6 +128,7 @@ ExtendedString as String, NoneType, NotKeywordSchema, + OAS3DefinitionHandler, OneOfCaseInsensitive, OneOfKeywordSchema, PermissiveMappingSchema, @@ -3936,6 +3938,7 @@ class ExecuteCollectionInput(FilterSchema, SortBySchema, PermissiveMappingSchema class ExecuteNestedProcessReference(ExtendedMappingSchema): + title = "ExecuteNestedProcessReference" # 'process' is required for a nested definition, otherwise it will not even be detected as one! process = ProcessURL(description="Process reference to be executed.") @@ -3950,12 +3953,18 @@ class ExecuteNestedProcessParameters(ExtendedMappingSchema): .. seealso:: - https://docs.pylonsproject.org/projects/colander/en/latest/binding.html """ + title = "ExecuteNestedProcessParameters" _sort_first = ["process", "inputs", "outputs", "properties", "mode", "response"] + _schema_extra = { + "type": null, + "title": "ExecuteNestedProcessParameters", + "$ref": f"{OAS3DefinitionHandler.json_pointer}ExecuteProcessParameters" + } @colander.deferred def _children(self, __bindings): # type: (Dict[str, Any]) -> List[colander.SchemaNode] - self.children = [node.clone() for node in ExecuteParameters().children] + self.children = [node.clone() for node in ExecuteProcessParameters().children] for child in self.children: # avoid inserting nested default properties that were omitted (ie: mode/response) # they should be included explicitly only on the top-most process by 'Execute(ExecuteParameters)' schema @@ -3983,6 +3992,7 @@ def deserialize(self, cstruct): class ExecuteNestedProcessInput(AllOfKeywordSchema): _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml" + title = "ExecuteNestedProcessInput" description = "Nested process to execute, for which the selected output will become the input of the parent call." _all_of = [ @@ -4002,6 +4012,7 @@ class ExecuteInputAnyType(OneOfKeywordSchema): """ Permissive variants that we attempt to parse automatically. """ + title = "ExecuteInputAnyType" _one_of = [ # Array of literal data with 'data' key ArrayLiteralDataType(), @@ -4023,12 +4034,16 @@ class ExecuteInputAnyType(OneOfKeywordSchema): ] -class ExecuteInputItem(ExecuteInputDataType, ExecuteInputAnyType): +class ExecuteInputItem(AllOfKeywordSchema): description = ( "Default value to be looked for uses key 'value' to conform to older drafts of OGC-API standard. " "Even older drafts that allowed other fields 'data' instead of 'value' and 'reference' instead of 'href' " "are also looked for to remain back-compatible." ) + _all_of = [ + ExecuteInputDataType(), + ExecuteInputAnyType(), + ] # backward compatible definition: @@ -4296,13 +4311,6 @@ class ExecuteParameters(ExecuteInputOutputs): These parameters can be either for a top-level process job, or any nested process call. """ _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml" - examples = { - "ExecuteJSON": { - "summary": "Execute a process job using REST JSON payload with OGC API schema.", - "value": EXAMPLES["job_execute.json"], - }, - } - title = JobTitle(missing=drop) mode = JobExecuteModeEnum( missing=drop, default=ExecuteMode.AUTO, @@ -4322,22 +4330,32 @@ class ExecuteParameters(ExecuteInputOutputs): subscribers = JobExecuteSubscribers(missing=drop) -class Execute(ExecuteParameters): - """ - Main execution parameters that can be submitted to run a process. - - Additional parameters are only applicable to the top-most process in a nested definition. - """ - # OGC 'execute.yaml' does not enforce any required item - description = "Process execution parameters." +class ExecuteProcessParameters(ExecuteParameters): + title = "ExecuteProcessParameters" _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml" + _sort_first = [ + "title", + "process", + "inputs", + "outputs", + "properties", + "mode", + "response", + "subscribers", + ] + _title = JobTitle(name="title", missing=drop) process = ProcessURL( missing=drop, description=( "Process reference to be executed. " "This parameter is required if the process cannot be inferred from the request endpoint." ), + example="https://example.com/processes/example" ) + + +class ExecuteJobParameters(ExtendedMappingSchema): + _title = JobTitle(name="title", missing=drop) status = JobStatusCreate( description=( "Status to request creation of the job without submitting it to processing queue " @@ -4348,6 +4366,38 @@ class Execute(ExecuteParameters): ) +class Execute(AllOfKeywordSchema): + """ + Main execution parameters that can be submitted to run a process. + + Additional parameters are only applicable to the top-most process in a nested definition. + """ + # OGC 'execute.yaml' does not enforce any required item + description = "Process execution parameters." + examples = { + "ExecuteJSON": { + "summary": "Execute a process job using REST JSON payload with OGC API schema.", + "value": EXAMPLES["job_execute.json"], + }, + } + _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/execute.yaml" + _sort_first = [ + "title", + "status", + "process", + "inputs", + "outputs", + "properties", + "mode", + "response", + "subscribers", + ] + _all_of = [ + ExecuteJobParameters(), + ExecuteProcessParameters(), + ] + + class QuoteStatusSchema(ExtendedSchemaNode): schema_type = String validator = OneOf(QuoteStatus.values())