From cf794a3ae05ee8bf675efae9261a8c56783fee7f Mon Sep 17 00:00:00 2001 From: fmigneault Date: Wed, 5 Jan 2022 21:51:02 -0500 Subject: [PATCH 01/34] fix lint --- magpie/ui/home/static/style.css | 1 - 1 file changed, 1 deletion(-) diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css index 39f060bdf..89d68acda 100644 --- a/magpie/ui/home/static/style.css +++ b/magpie/ui/home/static/style.css @@ -922,7 +922,6 @@ div.tree-button { text-align: left; margin-right: 0.5em; /* spacing with disallowed text to ensure one title doesn't end flush where the next begins */ width: 9.5em; /* [width = + permission-entry.width + permission-checkbox.width - margin-right] */ - /* make text >width employ an ellipsis with scroll that auto apply/remove when using it for too long titles */ text-overflow: ellipsis; overflow: auto; From 4de9fdbe96748977332fd0fda89f24b6d3a7dbe9 Mon Sep 17 00:00:00 2001 From: fmigneault Date: Wed, 5 Jan 2022 21:59:35 -0500 Subject: [PATCH 02/34] first implementation of geoserver multi-ows services --- magpie/models.py | 24 ++++-- magpie/services.py | 134 ++++++++++++++++++++++++++++---- magpie/ui/home/static/style.css | 1 + 3 files changed, 137 insertions(+), 22 deletions(-) diff --git a/magpie/models.py b/magpie/models.py index 410d8f5e8..bcf894035 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -362,7 +362,7 @@ def get(cls, status = status.split(",") if isinstance(status, (str, int)): return cls._get_one(status) - combined = None + combined = None # type: Optional[UserStatuses] for _status in status: _status = cls._get_one(_status) if combined is not None and _status is not None: @@ -406,6 +406,7 @@ def __xor__(self, other): # type: (Union[UserStatuses, int]) -> UserStatuses return super(UserStatuses, self).__xor__(other) def __iter__(self): + # type: () -> Iterable[UserStatuses] values = decompose_enum_flags(self) return iter(values) @@ -693,20 +694,28 @@ class Directory(Resource, PathBase): __mapper_args__ = {"polymorphic_identity": resource_type_name} +class Layer(Resource): + child_resource_allowed = False + resource_type_name = "layer" + __mapper_args__ = {"polymorphic_identity": resource_type_name} + + permissions = [ + Permission.GET_FEATURE, + Permission.DESCRIBE_FEATURE_TYPE, + Permission.LOCK_FEATURE, + Permission.TRANSACTION, + ] + + class Workspace(Resource): resource_type_name = "workspace" __mapper_args__ = {"polymorphic_identity": resource_type_name} permissions = [ - Permission.GET_CAPABILITIES, Permission.GET_MAP, Permission.GET_FEATURE_INFO, Permission.GET_LEGEND_GRAPHIC, Permission.GET_METADATA, - Permission.GET_FEATURE, - Permission.DESCRIBE_FEATURE_TYPE, - Permission.LOCK_FEATURE, - Permission.TRANSACTION, ] @@ -951,8 +960,9 @@ def json(self): RESOURCE_TREE_SERVICE = ResourceTreeService(ResourceTreeServicePostgreSQL) REMOTE_RESOURCE_TREE_SERVICE = RemoteResourceTreeService(RemoteResourceTreeServicePostgresSQL) +RESOURCE_TYPES = frozenset([Service, Directory, File, Layer, Workspace, Route, Process]) RESOURCE_TYPE_DICT = dict() # type: Dict[Str, Type[Resource]] -for res in [Service, Directory, File, Workspace, Route, Process]: +for res in RESOURCE_TYPES: if res.resource_type_name in RESOURCE_TYPE_DICT: # pragma: no cover raise KeyError("Duplicate resource type identifiers not allowed") RESOURCE_TYPE_DICT[res.resource_type_name] = res diff --git a/magpie/services.py b/magpie/services.py index 8901d3b63..e13bb4138 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -1,4 +1,5 @@ import abc +import inspect import re from typing import TYPE_CHECKING @@ -582,13 +583,11 @@ class ServiceWPS(ServiceOWS): ] resource_types_permissions = { - models.Process: [ - Permission.DESCRIBE_PROCESS, - Permission.EXECUTE, - ] + models.Process: models.Process.permissions } def resource_requested(self): + # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] wps_request = self.permission_requested() if wps_request == Permission.GET_CAPABILITIES: return self.service, True @@ -656,6 +655,7 @@ class ServiceNCWMS2(ServiceBaseWMS): } def resource_requested(self): + # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] # According to the permission, the resource we want to authorize is not formatted the same way permission_requested = self.permission_requested() netcdf_file = None @@ -709,16 +709,13 @@ class ServiceGeoserverWMS(ServiceBaseWMS): service_type = "geoserverwms" resource_types_permissions = { - models.Workspace: [ - Permission.GET_CAPABILITIES, - Permission.GET_MAP, - Permission.GET_FEATURE_INFO, - Permission.GET_LEGEND_GRAPHIC, - Permission.GET_METADATA, - ] + # workspace must allow permissions for layers as well as parent in hierarchy + models.Workspace: models.Workspace.permissions + models.Layer.permissions, + models.Layer: models.Layer.permissions } def resource_requested(self): + # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] path_parts = self._get_request_path_parts() if not path_parts: return self.service, False @@ -789,7 +786,6 @@ class ServiceAPI(ServiceInterface): def resource_requested(self): # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] - route_parts = self._get_request_path_parts() if not route_parts: return self.service, True @@ -907,6 +903,7 @@ def is_match(value, pattern): return None def resource_requested(self): + # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] path_parts = self.get_path_parts() # handle optional prefix as targeting the service directly @@ -959,11 +956,118 @@ def permission_requested(self): return None # automatically deny +class ServiceGeoserverMeta(ServiceMeta): + """ + Mapping and grouping of property definitions for ``GeoServer`` services from distinct `OWS` implementations. + """ + service_map = { + "wfs": ServiceWFS, + "wms": ServiceGeoserverWMS, + "wps": ServiceWPS, + "WFS": ServiceWFS, + "WMS": ServiceGeoserverWMS, + "WPS": ServiceWPS, + } + + @property + def permissions(self): + # type: () -> List[Permission] + perms = set() + for svc in self.supported_ows: + if issubclass(svc, ServiceOWS) and hasattr(svc, "permissions"): + perms.update(svc.permissions) + return list(perms) + + @property + def resource_types_permissions(self): + # type: () -> Dict[models.Resource, List[Permission]] + perms = {} + for svc in self.supported_ows: + if issubclass(svc, ServiceOWS) and hasattr(svc, "resource_types_permissions"): + perms.update(svc.resource_types_permissions) + return perms + + @property + def supported_ows(self): + # type: () -> Set[Type[ServiceOWS]] + return set(self.service_map.values()) + + +@six.add_metaclass(ServiceGeoserverMeta) +class ServiceGeoserver(ServiceOWS): + """ + Service that encapsulates the multiple `OWS` endpoints from ``GeoServer`` services. + + .. seealso:: + https://docs.geoserver.org/stable/en/user/services/index.html + """ + service_type = "geoserver" + + params_expected = [ + "request", + "service" + ] + + def service_requested(self): + # type: () -> Type[ServiceOWS] + """ + Obtain the applicable `OWS` implementation according to parsed request parameters. + """ + try: + svc = self.parser.params["service"] + except KeyError: + svc = None + req = self.parser.params.get("request") + if svc is None and req is not None: + # geoserver allows omitting 'service' request parameter because it can be inferred from the path + # since all OWS services are accessed using '/geoserver/?request=...' + # attempt to match using applicable 'request' parameter + for geo_svc in self.supported_ows: + if issubclass(geo_svc, ServiceInterface) and hasattr(geo_svc, "permissions"): + perm = Permission(req) + if perm in geo_svc.permissions: + svc = geo_svc + break + ax.verify_param( + svc, is_in=True, param_compare=self.service_map, param_name="service", http_error=HTTPBadRequest, + content={ + "service": self.service.resource_name, + "type": self.service_type, + "value": {"service": svc, "request": req} + }, + msg_on_fail=( + "Missing or unknown implementation inferred from OWS 'service' and 'request' parameters. " + + "Unable to resolve the requested access for service: [{!s}].".format(self.service.resource_name) + ) + ) + return self.service_map[svc] + + def resource_requested(self): + # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] + svc = self.service_requested() + return svc(self.service, self.request).resource_requested() + + def permission_requested(self): + # type: () -> Permission + svc = self.service_requested() + return svc(self.service, self.request).permission_requested() + + +SERVICE_TYPES = frozenset([ + ServiceAccess, + ServiceAPI, + ServiceGeoserver, + ServiceGeoserverWMS, + ServiceNCWMS2, + ServiceTHREDDS, + ServiceWFS, + ServiceWPS +]) SERVICE_TYPE_DICT = dict() -for svc in [ServiceAccess, ServiceAPI, ServiceGeoserverWMS, ServiceNCWMS2, ServiceTHREDDS, ServiceWFS, ServiceWPS]: - if svc.service_type in SERVICE_TYPE_DICT: +for _svc in SERVICE_TYPES: + if _svc.service_type in SERVICE_TYPE_DICT: raise KeyError("Duplicate resource type identifiers not allowed") - SERVICE_TYPE_DICT[svc.service_type] = svc + SERVICE_TYPE_DICT[_svc.service_type] = _svc def service_factory(service, request): diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css index 89d68acda..39f060bdf 100644 --- a/magpie/ui/home/static/style.css +++ b/magpie/ui/home/static/style.css @@ -922,6 +922,7 @@ div.tree-button { text-align: left; margin-right: 0.5em; /* spacing with disallowed text to ensure one title doesn't end flush where the next begins */ width: 9.5em; /* [width = + permission-entry.width + permission-checkbox.width - margin-right] */ + /* make text >width employ an ellipsis with scroll that auto apply/remove when using it for too long titles */ text-overflow: ellipsis; overflow: auto; From da2bbf8a79c1c40778b4e062cf1d1413f3524968 Mon Sep 17 00:00:00 2001 From: fmigneault Date: Thu, 6 Jan 2022 12:49:59 -0500 Subject: [PATCH 03/34] adjust UI to improve display within edit service page when many permissions are applicable --- magpie/ui/home/static/style.css | 6 ++++++ magpie/ui/management/templates/edit_service.mako | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css index 39f060bdf..cece10d32 100644 --- a/magpie/ui/home/static/style.css +++ b/magpie/ui/home/static/style.css @@ -382,6 +382,12 @@ table.panel-line td { */ } +.panel-line-limit-size { + /* reduce size of the field to allow other fields to preserve 'Edit' button closer to their value + in case the large value contents within this field makes the table column very wide */ + max-width: 1em; +} + .panel-line-checkbox { padding-top: 0.1em; display: inline-block; diff --git a/magpie/ui/management/templates/edit_service.mako b/magpie/ui/management/templates/edit_service.mako index 37268521d..07f1e7855 100644 --- a/magpie/ui/management/templates/edit_service.mako +++ b/magpie/ui/management/templates/edit_service.mako @@ -154,7 +154,7 @@ Permissions: -
+
%for perm in service_perm: ${perm} %endfor From 1e96bb0caa6b0e2a5d4834b35bccff40480e3b64 Mon Sep 17 00:00:00 2001 From: fmigneault Date: Thu, 6 Jan 2022 12:51:47 -0500 Subject: [PATCH 04/34] =?UTF-8?q?Bump=20version:=203.19.1=20=E2=86=92=203.?= =?UTF-8?q?20.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.rst | 5 +++++ Makefile | 2 +- README.rst | 18 +++++++++--------- magpie/__meta__.py | 2 +- setup.cfg | 2 +- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ff55ae7d6..5a335e785 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,11 @@ Changes `Unreleased `_ (latest) ------------------------------------------------------------------------------------ +* Nothing new for the moment. + +`3.20.0 `_ (2022-01-06) +------------------------------------------------------------------------------------ + Features / Changes ~~~~~~~~~~~~~~~~~~~~~ * Add improved UI display of long ``Permission`` titles for ``Resource`` hierarchy tree headers. diff --git a/Makefile b/Makefile index 58ed516e1..b742f84db 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) # Application APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..) APP_NAME := magpie -APP_VERSION ?= 3.19.1 +APP_VERSION ?= 3.20.0 APP_INI ?= $(APP_ROOT)/config/$(APP_NAME).ini # guess OS (Linux, Darwin,...) diff --git a/README.rst b/README.rst index 818bebfab..c9160a86f 100644 --- a/README.rst +++ b/README.rst @@ -29,13 +29,13 @@ Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. :alt: Requires Python 2.7, 3.5+ :target: https://www.python.org/getit -.. |commits-since| image:: https://img.shields.io/github/commits-since/Ouranosinc/Magpie/3.19.1.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/Ouranosinc/Magpie/3.20.0.svg :alt: Commits since latest release - :target: https://github.com/Ouranosinc/Magpie/compare/3.19.1...master + :target: https://github.com/Ouranosinc/Magpie/compare/3.20.0...master -.. |version| image:: https://img.shields.io/badge/tag-3.19.1-blue.svg?style=flat +.. |version| image:: https://img.shields.io/badge/tag-3.20.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/Ouranosinc/Magpie/tree/3.19.1 + :target: https://github.com/Ouranosinc/Magpie/tree/3.20.0 .. |dependencies| image:: https://pyup.io/repos/github/Ouranosinc/Magpie/shield.svg :alt: Dependencies Status @@ -45,9 +45,9 @@ Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. :alt: Github Actions CI Build Status (master branch) :target: https://github.com/Ouranosinc/Magpie/actions?query=workflow%3ATests+branch%3Amaster -.. |github_tagged| image:: https://img.shields.io/github/workflow/status/Ouranosinc/Magpie/Tests/3.19.1?label=3.19.1 +.. |github_tagged| image:: https://img.shields.io/github/workflow/status/Ouranosinc/Magpie/Tests/3.20.0?label=3.20.0 :alt: Github Actions CI Build Status (latest tag) - :target: https://github.com/Ouranosinc/Magpie/actions?query=workflow%3ATests+branch%3A3.19.1 + :target: https://github.com/Ouranosinc/Magpie/actions?query=workflow%3ATests+branch%3A3.20.0 .. |readthedocs| image:: https://img.shields.io/readthedocs/pavics-magpie :alt: Readthedocs Build Status (master branch) @@ -75,7 +75,7 @@ Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. .. |docker_semver_tag| image:: https://img.shields.io/docker/v/pavics/magpie?label=version&sort=semver :alt: Docker Version Tag - :target: https://hub.docker.com/r/pavics/magpie/tags?page=1&ordering=last_updated&name=3.19.1 + :target: https://hub.docker.com/r/pavics/magpie/tags?page=1&ordering=last_updated&name=3.20.0 .. end-badges @@ -119,8 +119,8 @@ Following most recent variants are available: * - Magpie - Twitcher |br| (with integrated ``MagpieAdapter``) - * - ``pavics/magpie:3.19.1`` - - ``pavics/twitcher:magpie-3.19.1`` + * - ``pavics/magpie:3.20.0`` + - ``pavics/twitcher:magpie-3.20.0`` * - ``pavics/magpie:latest`` - ``pavics/twitcher:magpie-latest`` diff --git a/magpie/__meta__.py b/magpie/__meta__.py index c586fa12c..06de0d8a1 100644 --- a/magpie/__meta__.py +++ b/magpie/__meta__.py @@ -2,7 +2,7 @@ General meta information on the magpie package. """ -__version__ = "3.19.1" +__version__ = "3.20.0" __title__ = "Magpie" __package__ = "magpie" # pylint: disable=W0622 __author__ = "Francois-Xavier Derue, Francis Charette-Migneault" diff --git a/setup.cfg b/setup.cfg index d6ec49f9f..117538dad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.19.1 +current_version = 3.20.0 commit = True tag = True tag_name = {new_version} From f65683165db548c41c2a715daf9d6de18bdc7505 Mon Sep 17 00:00:00 2001 From: fmigneault Date: Thu, 6 Jan 2022 15:04:46 -0500 Subject: [PATCH 05/34] allowed scoped resource names using colon character --- magpie/api/exception.py | 1 + magpie/api/management/resource/resource_utils.py | 1 + magpie/api/management/resource/resource_views.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/magpie/api/exception.py b/magpie/api/exception.py index d33e3884e..2c59c197f 100644 --- a/magpie/api/exception.py +++ b/magpie/api/exception.py @@ -44,6 +44,7 @@ # utility parameter validation regexes for 'matches' argument PARAM_REGEX = r"^[A-Za-z0-9]+(?:[\s_\-\.][A-Za-z0-9]+)*$" # request parameters +SCOPE_REGEX = r"^[A-Za-z0-9]+(?:[\:\s_\-\.][A-Za-z0-9]+)*$" # allow scoped names (e.g.: 'namespace:value') EMAIL_REGEX = colander.EMAIL_RE UUID_REGEX = colander.UUID_REGEX URL_REGEX = colander.URL_REGEX diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py index 0e3d493ff..82f212f69 100644 --- a/magpie/api/management/resource/resource_utils.py +++ b/magpie/api/management/resource/resource_utils.py @@ -273,6 +273,7 @@ def get_resource_root_service_impl(resource, request): def create_resource(resource_name, resource_display_name, resource_type, parent_id, db_session): # type: (Str, Optional[Str], Str, int, Session) -> HTTPException ax.verify_param(resource_name, param_name="resource_name", not_none=True, not_empty=True, + matches=True, param_compare=ax.SCOPE_REGEX, http_error=HTTPUnprocessableEntity, msg_on_fail="Invalid 'resource_name' specified for child resource creation.") ax.verify_param(resource_type, param_name="resource_type", not_none=True, not_empty=True, diff --git a/magpie/api/management/resource/resource_views.py b/magpie/api/management/resource/resource_views.py index 1cfef30e9..bb863288a 100644 --- a/magpie/api/management/resource/resource_views.py +++ b/magpie/api/management/resource/resource_views.py @@ -55,7 +55,7 @@ def create_resource_view(request): """ Register a new resource. """ - resource_name = ar.get_value_multiformat_body_checked(request, "resource_name") + resource_name = ar.get_multiformat_body(request, "resource_name") resource_display_name = ar.get_multiformat_body(request, "resource_display_name", default=resource_name) resource_type = ar.get_value_multiformat_body_checked(request, "resource_type") parent_id = ar.get_value_multiformat_body_checked(request, "parent_id", check_type=int) From 9b04a1e3fd5802781dc309eba609548ec448c764 Mon Sep 17 00:00:00 2001 From: fmigneault Date: Thu, 6 Jan 2022 15:48:27 -0500 Subject: [PATCH 06/34] add tests to validate proper support of scoped names for service and resources --- CHANGES.rst | 4 ++- .../api/management/service/service_utils.py | 2 +- .../api/management/service/service_views.py | 5 ++- magpie/api/requests.py | 2 +- magpie/api/schemas.py | 5 +++ tests/interfaces.py | 33 +++++++++++++++++++ tests/utils.py | 2 +- 7 files changed, 48 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5a335e785..c7e1adb6b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,7 +7,9 @@ Changes `Unreleased `_ (latest) ------------------------------------------------------------------------------------ -* Nothing new for the moment. +Features / Changes +~~~~~~~~~~~~~~~~~~~~~ +* Allow ``Resource`` and ``Service`` name to contain colon (``:``) character in order to define scoped names. `3.20.0 `_ (2022-01-06) ------------------------------------------------------------------------------------ diff --git a/magpie/api/management/service/service_utils.py b/magpie/api/management/service/service_utils.py index 75e0c29f4..dd5ea2426 100644 --- a/magpie/api/management/service/service_utils.py +++ b/magpie/api/management/service/service_utils.py @@ -67,7 +67,7 @@ def _add_service_magpie_and_phoenix(svc, svc_push, db): ax.verify_param(service_url, matches=True, param_compare=ax.URL_REGEX, param_name="service_url", http_error=HTTPBadRequest, msg_on_fail=s.Services_POST_Params_BadRequestResponseSchema.description) ax.verify_param(service_name, not_empty=True, not_none=True, matches=True, - param_name="service_name", param_compare=ax.PARAM_REGEX, + param_name="service_name", param_compare=ax.SCOPE_REGEX, http_error=HTTPBadRequest, msg_on_fail=s.Services_POST_Params_BadRequestResponseSchema.description) ax.verify_param(models.Service.by_service_name(service_name, db_session=db_session), is_none=True, param_name="service_name", with_param=False, content={"service_name": str(service_name)}, diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index b82cc852f..cbfbc852a 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -113,7 +113,7 @@ def register_service_view(request): Registers a new service. """ # accomplish basic validations here, create_service will do more field-specific checks - service_name = ar.get_value_multiformat_body_checked(request, "service_name") + service_name = ar.get_value_multiformat_body_checked(request, "service_name", pattern=ax.SCOPE_REGEX) service_url = ar.get_value_multiformat_body_checked(request, "service_url", pattern=ax.URL_REGEX) service_type = ar.get_value_multiformat_body_checked(request, "service_type") service_push = asbool(ar.get_multiformat_body(request, "service_push", default=False)) @@ -166,6 +166,9 @@ def select_update(new_value, old_value): ax.verify_param(svc_name, not_in=True, param_compare=all_svc_names, with_param=False, http_error=HTTPConflict, content={"service_name": str(svc_name)}, msg_on_fail=s.Service_PATCH_ConflictResponseSchema.description) + ax.verify_param(svc_name, not_none=True, not_empty=True, matches=True, param_compare=ax.SCOPE_REGEX, + http_error=HTTPBadRequest, + msg_on_fail=s.Service_PATCH_UnprocessableEntityResponseSchema.description) def update_service_magpie_and_phoenix(_svc, new_name, new_url, svc_push, db_session): _svc.resource_name = new_name diff --git a/magpie/api/requests.py b/magpie/api/requests.py index cf0ea2f94..1ef60ccab 100644 --- a/magpie/api/requests.py +++ b/magpie/api/requests.py @@ -311,7 +311,7 @@ def get_service_matchdict_checked(request, service_name_key="service_name"): :raises HTTPForbidden: if the requesting user does not have sufficient permission to execute this request. :raises HTTPNotFound: if the specified service name does not correspond to any existing service. """ - service_name = get_value_matchdict_checked(request, service_name_key) + service_name = get_value_matchdict_checked(request, service_name_key, pattern=ax.SCOPE_REGEX) service = ax.evaluate_call(lambda: models.Service.by_service_name(service_name, db_session=request.db), fallback=lambda: request.db.rollback(), http_error=HTTPForbidden, msg_on_fail=s.Service_MatchDictCheck_ForbiddenResponseSchema.description) diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index 3ac19e8a1..9f32daef6 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -1509,6 +1509,11 @@ class Service_PATCH_ConflictResponseSchema(BaseResponseSchemaAPI): body = ErrorResponseBodySchema(code=HTTPConflict.code, description=description) +class Service_PATCH_UnprocessableEntityResponseSchema(BaseResponseSchemaAPI): + description = "Specified value is in incorrectly or unsupported format." + body = ErrorResponseBodySchema(code=HTTPUnprocessableEntity.code, description=description) + + Service_DELETE_RequestBodySchema = Resource_DELETE_RequestBodySchema diff --git a/tests/interfaces.py b/tests/interfaces.py index bd1970327..151234366 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -5636,6 +5636,39 @@ def test_PostServiceResources_DirectResource_Conflict(self): is_param_value_literal_unicode=True, param_compare_exists=True, param_value=self.test_resource_name, param_name="resource_name") + @runner.MAGPIE_TEST_RESOURCES + def test_PostServiceResources_ScopedName(self): + """ + Validate that scoped names are allowed for :term:`Service` and :term:`Resource` operations. + """ + utils.warn_version(self, "resource name with scoping", "3.21.0", skip=True) + scope = "test:" + scope_svc = scope + self.test_service_name + scope_res = scope + self.test_resource_name + utils.TestSetup.delete_TestService(self, override_service_name=scope_svc) + body = utils.TestSetup.create_TestServiceResource(self, + override_service_name=scope_svc, + override_resource_name=scope_res) + res_info = utils.TestSetup.get_ResourceInfo(self, override_body=body, full_detail=True) + utils.check_val_equal(res_info["resource_name"], scope_res) + + # explicit check of GET to validate scoped names in path are supported + svc_info = utils.TestSetup.get_ExistingTestServiceInfo(self, override_service_name=scope_svc) + utils.check_val_equal(svc_info["service_name"], scope_svc) + + # check that second POST also properly detects scoped resource name + data = { + "resource_name": scope_res, + "resource_type": self.test_resource_type, + "parent_id": res_info["parent_id"], + } + resp = utils.test_request(self, "POST", "/resources", json=data, expect_errors=True, + headers=self.json_headers, cookies=self.cookies) + utils.check_response_basic_info(resp, 409, expected_method="POST") + + # explicit check of DELETE to validate it is also supported + utils.TestSetup.delete_TestService(self, override_service_name=scope_svc) + @runner.MAGPIE_TEST_SERVICES @runner.MAGPIE_TEST_DEFAULTS def test_ValidateDefaultServiceProviders(self): diff --git a/tests/utils.py b/tests/utils.py index 8a3c78290..ea47694f1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1284,7 +1284,7 @@ def check_val_not_in(val, ref, msg=None): def check_val_type(val, ref, msg=None): - # type: (Any, Union[Type[Any], NullType], Optional[Str]) -> None + # type: (Any, Union[Type[Any], Tuple[Type[Any]], NullType], Optional[Str]) -> None """:raises AssertionError: if :paramref:`val` is not an instanced of :paramref:`ref`.""" assert isinstance(val, ref), format_test_val_ref(val, repr(ref), pre="Type Fail", msg=msg) From e078ace8f890c6b1395b9e58df5760b5b12b8c65 Mon Sep 17 00:00:00 2001 From: fmigneault Date: Fri, 7 Jan 2022 00:27:57 -0500 Subject: [PATCH 07/34] support specific nested structure of children resource types --- CHANGES.rst | 20 ++ magpie/api/management/group/__init__.py | 1 - magpie/api/management/resource/__init__.py | 1 + .../api/management/resource/resource_utils.py | 23 ++- .../api/management/resource/resource_views.py | 43 ++++ .../api/management/service/service_formats.py | 8 +- magpie/api/management/user/__init__.py | 2 - magpie/api/schemas.py | 100 ++++++++-- magpie/services.py | 186 +++++++++++++++--- magpie/ui/management/views.py | 4 +- 10 files changed, 339 insertions(+), 49 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c7e1adb6b..58b9e5122 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,26 @@ Changes Features / Changes ~~~~~~~~~~~~~~~~~~~~~ * Allow ``Resource`` and ``Service`` name to contain colon (``:``) character in order to define scoped names. +* Add ``child_structure_allowed`` attribute to ``Service`` implementations allowing them to define specific path-like + structures of allowed ``Resource`` types hierarchies in order to control at which level and which combinations + of nested ``Resource`` types are valid under their root ``Service``. When not defined under a ``Service`` + implementation, any defined ``Resource`` type will remain available for creation at any level of the hierarchy, + unless the corresponding ``Resource`` in the tree already defined ``child_resource_allowed = False``. This was + already the original behaviour in previous versions. +* Add ``GET /resources/{id}/types`` endpoint that allows retrieval of applicable children ``Resource`` types under + a given ``Resource`` considering the nested hierarchy definition of its root ``Service`` defined by the new + attribute ``child_structure_allowed``. +* Add ``child_structure_allowed`` attribute to the response of ``GET /service/{name}`` endpoint. + For backward compatibility, ``resource_types_allowed`` parameter already available in the same response will continue + to report all possible ``Resource`` types *at any level* under the ``Service`` hierarchy, although not necessarily + applicable as immediate child ``Resource`` under that ``Service``. +* Adjust UI to consider ``child_structure_allowed`` definitions to propose only applicable ``Resource`` types in the + combobox when creating a new ``Resource`` in the tree hierarchy. + +Bug Fixes +~~~~~~~~~~~~~~~~~~~~~ +* Remove invalid ``request`` parameter in ``ServiceTHREDDS`` implementation. +* Remove multiple invalid schema path definitions that are not mapped against any concrete API endpoint. `3.20.0 `_ (2022-01-06) ------------------------------------------------------------------------------------ diff --git a/magpie/api/management/group/__init__.py b/magpie/api/management/group/__init__.py index 4bc03205a..0d84b184a 100644 --- a/magpie/api/management/group/__init__.py +++ b/magpie/api/management/group/__init__.py @@ -17,6 +17,5 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.GroupResourcesAPI)) config.add_route(**s.service_api_route_info(s.GroupResourcePermissionsAPI)) config.add_route(**s.service_api_route_info(s.GroupResourcePermissionAPI)) - config.add_route(**s.service_api_route_info(s.GroupResourceTypesAPI)) config.scan() diff --git a/magpie/api/management/resource/__init__.py b/magpie/api/management/resource/__init__.py index 2c30e2d67..e8582980a 100644 --- a/magpie/api/management/resource/__init__.py +++ b/magpie/api/management/resource/__init__.py @@ -10,5 +10,6 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.ResourcesAPI)) config.add_route(**s.service_api_route_info(s.ResourceAPI)) config.add_route(**s.service_api_route_info(s.ResourcePermissionsAPI)) + config.add_route(**s.service_api_route_info(s.ResourceTypesAPI)) config.scan() diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py index 82f212f69..cdb978985 100644 --- a/magpie/api/management/resource/resource_utils.py +++ b/magpie/api/management/resource/resource_utils.py @@ -58,6 +58,7 @@ def check_valid_service_or_resource_permission(permission_name, service_or_resou def check_valid_service_resource(parent_resource, resource_type, db_session): + # type: (ServiceOrResourceType, Str, Session) -> models.Service """ Checks if a new Resource can be contained under a parent Resource given the requested type and the corresponding Service under which the parent Resource is already assigned. @@ -69,20 +70,32 @@ def check_valid_service_resource(parent_resource, resource_type, db_session): """ parent_type = parent_resource.resource_type_name parent_msg_err = "Child resource not allowed for specified parent resource type '{}'".format(parent_type) - ax.verify_param(models.RESOURCE_TYPE_DICT[parent_type].child_resource_allowed, is_equal=True, - param_compare=True, http_error=HTTPForbidden, msg_on_fail=parent_msg_err) + ax.verify_param(models.RESOURCE_TYPE_DICT[parent_type].child_resource_allowed, is_true=True, + http_error=HTTPForbidden, msg_on_fail=parent_msg_err) root_service = get_resource_root_service(parent_resource, db_session=db_session) ax.verify_param(root_service, not_none=True, http_error=HTTPInternalServerError, msg_on_fail="Failed retrieving 'root_service' from db") ax.verify_param(root_service.resource_type, is_equal=True, http_error=HTTPInternalServerError, param_name="resource_type", param_compare=models.Service.resource_type_name, msg_on_fail="Invalid 'root_service' retrieved from db is not a service") - ax.verify_param(SERVICE_TYPE_DICT[root_service.type].child_resource_allowed, is_equal=True, - param_compare=True, http_error=HTTPForbidden, + root_svc_cls = SERVICE_TYPE_DICT[root_service.type] + ax.verify_param(root_svc_cls.child_resource_allowed, is_true=True, http_error=HTTPForbidden, msg_on_fail="Child resource not allowed for specified service type '{}'".format(root_service.type)) ax.verify_param(resource_type, is_in=True, http_error=HTTPForbidden, - param_name="resource_type", param_compare=SERVICE_TYPE_DICT[root_service.type].resource_type_names, + param_name="resource_type", param_compare=root_svc_cls.resource_type_names, msg_on_fail="Invalid 'resource_type' specified for service type '{}'".format(root_service.type)) + ax.verify_param(root_svc_cls.validate_nested_resource_type(parent_resource, resource_type), is_true=True, + param_content={ + "resource_structure_allowed": root_svc_cls.child_structure_allowed, + "resource_types_allowed": [ + res.resource_type for res in root_svc_cls.nested_resource_allowed(parent_resource) + ] + }, + http_error=HTTPUnprocessableEntity, + msg_on_fail=( + "Invalid 'resource_type' specified for service type '{}' is not allowed at this position " + "under '{}' resource.".format(root_service.type, parent_type) + )) return root_service diff --git a/magpie/api/management/resource/resource_views.py b/magpie/api/management/resource/resource_views.py index bb863288a..59fcc438a 100644 --- a/magpie/api/management/resource/resource_views.py +++ b/magpie/api/management/resource/resource_views.py @@ -127,3 +127,46 @@ def get_resource_permissions_view(request): content={"resource": rf.format_resource(resource, basic_info=True)}) return ax.valid_http(http_success=HTTPOk, detail=s.ResourcePermissions_GET_OkResponseSchema.description, content=format_permissions(res_perm, PermissionType.ALLOWED)) + + +@s.ResourceTypesAPI.get(schema=s.ResourceTypes_GET_RequestSchema, tags=[s.ResourcesTag], + response_schemas=s.ResourceTypes_GET_responses) +@view_config(route_name=s.ResourceTypesAPI.name, request_method="GET") +def get_resource_types_view(request): + """ + List all applicable children resource types under another resource within a service hierarchy. + """ + resource = ar.get_resource_matchdict_checked(request, "resource_id") + + def get_res_types(res): + svc_root = ru.get_resource_root_service(res, db_session=request.db) + svc_impl = SERVICE_TYPE_DICT[svc_root.type] + return svc_impl.nested_resource_allowed(res), svc_root + + def get_res_child_allowed(res): + # make sure to obtain the specific resource/service implementation to avoid using the default + if res.resource_type_name == models.Service.resource_type_name: + res_impl = SERVICE_TYPE_DICT[res.type] + else: + res_impl = models.RESOURCE_TYPE_DICT[res.resource_type_name] + return res_impl.child_resource_allowed + + res_types, svc = ax.evaluate_call(lambda: get_res_types(resource), + fallback=lambda: request.db.rollback(), http_error=HTTPInternalServerError, + msg_on_fail="Error occurred while computing applicable children resource types.", + content={"resource": rf.format_resource(resource, basic_info=True)}) + child_allowed = ax.evaluate_call(lambda: get_res_child_allowed(resource), + http_error=HTTPInternalServerError, + msg_on_fail="Error occurred while computing allowed children resource status.", + content={"resource": rf.format_resource(resource, basic_info=True)}) + data = { + "resource_name": resource.resource_name, + "resource_type": resource.resource_type_name, + "children_resource_types": list(sorted(res_type.resource_type_name for res_type in res_types)), + "children_resource_allowed": child_allowed, + "root_service_id": svc.resource_id, + "root_service_name": svc.resource_name, + "root_service_type": svc.type, + } + return ax.valid_http(http_success=HTTPOk, content=data, + detail=s.ResourceTypes_GET_OkResponseSchema.description) diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py index 9e0d2eb5c..6b0bcf271 100644 --- a/magpie/api/management/service/service_formats.py +++ b/magpie/api/management/service/service_formats.py @@ -76,11 +76,13 @@ def fmt_svc(): return svc_info if show_configuration: svc_info["configuration"] = service.configuration - perms = SERVICE_TYPE_DICT[service.type].permissions if permissions is None else permissions + svc_type = SERVICE_TYPE_DICT[service.type] + perms = svc_type.permissions if permissions is None else permissions svc_info.update(format_permissions(perms, permission_type)) if show_resources_allowed: - svc_info["resource_types_allowed"] = sorted(SERVICE_TYPE_DICT[service.type].resource_type_names) - svc_info["resource_child_allowed"] = SERVICE_TYPE_DICT[service.type].child_resource_allowed + svc_info["resource_child_allowed"] = svc_type.child_resource_allowed + svc_info["resource_types_allowed"] = sorted(svc_type.resource_type_names) + svc_info["resource_structure_allowed"] = sorted(svc_type.child_structure_allowed) return svc_info return evaluate_call( diff --git a/magpie/api/management/user/__init__.py b/magpie/api/management/user/__init__.py index bf9691f2e..bd9e47b3b 100644 --- a/magpie/api/management/user/__init__.py +++ b/magpie/api/management/user/__init__.py @@ -21,7 +21,6 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.UserServicePermissionAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.UserServiceResourcesAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.UserResourcesAPI, **user_kwargs)) - config.add_route(**s.service_api_route_info(s.UserResourceTypesAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.UserResourcePermissionsAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.UserResourcePermissionAPI, **user_kwargs)) # Logged User routes @@ -33,7 +32,6 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.LoggedUserServicePermissionAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.LoggedUserServiceResourcesAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.LoggedUserResourcesAPI, **user_kwargs)) - config.add_route(**s.service_api_route_info(s.LoggedUserResourceTypesAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.LoggedUserResourcePermissionsAPI, **user_kwargs)) config.add_route(**s.service_api_route_info(s.LoggedUserResourcePermissionAPI, **user_kwargs)) diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index 9f32daef6..c9ce16e6d 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -122,9 +122,6 @@ def service_api_route_info(service_api, **kwargs): UserResourcePermissionAPI = Service( path="/users/{user_name}/resources/{resource_id}/permissions/{permission_name}", name="UserResourcePermission") -UserResourceTypesAPI = Service( - path="/users/{user_name}/resources/types/{resource_type}", - name="UserResourceTypes") UserServicesAPI = Service( path="/users/{user_name}/services", name="UserServices") @@ -158,9 +155,6 @@ def service_api_route_info(service_api, **kwargs): LoggedUserResourcePermissionsAPI = Service( path=LoggedUserBase + "/resources/{resource_id}/permissions", name="LoggedUserResourcePermissions") -LoggedUserResourceTypesAPI = Service( - path=LoggedUserBase + "/resources/types/{resource_type}", - name="LoggedUserResourceTypes") LoggedUserServicesAPI = Service( path=LoggedUserBase + "/services", name="LoggedUserServices") @@ -203,9 +197,6 @@ def service_api_route_info(service_api, **kwargs): GroupResourcePermissionAPI = Service( path="/groups/{group_name}/resources/{resource_id}/permissions/{permission_name}", name="GroupResourcePermission") -GroupResourceTypesAPI = Service( - path="/groups/{group_name}/resources/types/{resource_type}", - name="GroupResourceTypes") RegisterGroupsAPI = Service( path="/register/groups", name="RegisterGroups") @@ -227,6 +218,9 @@ def service_api_route_info(service_api, **kwargs): ResourcePermissionsAPI = Service( path="/resources/{resource_id}/permissions", name="ResourcePermissions") +ResourceTypesAPI = Service( + path="/resources/{resource_id}/types", + name="ResourceTypes") ServicesAPI = Service( path="/services", name="Services") @@ -713,6 +707,14 @@ class ResourceTypesListSchema(colander.SequenceSchema): ) +class ChildrenResourceTypesListSchema(colander.SequenceSchema): + resource_type = colander.SchemaNode( + colander.String(), + description="Available children resource types under a resource within a service hierarchy.", + example="file", + ) + + class GroupNamesListSchema(colander.SequenceSchema): group_name = GroupNameParameter @@ -1147,7 +1149,7 @@ class Resources_GET_OkResponseSchema(BaseResponseSchemaAPI): class Resources_POST_RequestBodySchema(colander.MappingSchema): resource_name = colander.SchemaNode( colander.String(), - description="Name of the resource to create" + description="Name of the resource to create." ) resource_display_name = colander.SchemaNode( colander.String(), @@ -1156,11 +1158,11 @@ class Resources_POST_RequestBodySchema(colander.MappingSchema): ) resource_type = colander.SchemaNode( colander.String(), - description="Type of the resource to create" + description="Type of the resource to create." ) parent_id = colander.SchemaNode( colander.Int(), - description="ID of parent resource under which the new resource should be created", + description="ID of parent resource under which the new resource should be created.", missing=colander.drop ) @@ -1222,6 +1224,51 @@ class ResourcePermissions_GET_BadRequestResponseSchema(BaseResponseSchemaAPI): body = ErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) +class ResourceTypes_GET_RequestSchema(BaseRequestSchemaAPI): + path = Resource_RequestPathSchema() + + +class ResourceTypes_GET_ResponseBodySchema(BaseResponseBodySchema): + resource_name = colander.SchemaNode( + colander.String(), + description="Name of the specified resource." + ) + resource_type = colander.SchemaNode( + colander.String(), + description="Type of the specified resource." + ) + children_resource_types = ChildrenResourceTypesListSchema( + description="All resource types applicable as children under the specified resource. " + ) + children_resource_allowed = colander.SchemaNode( + colander.Boolean(), + description="Indicates if the resource allows any children resources." + ) + root_service_id = colander.SchemaNode( + colander.Integer(), + description="Resource tree root service identification number.", + default=colander.null, # if no parent + missing=colander.drop # if not returned (basic_info = True) + ) + root_service_name = colander.SchemaNode( + colander.Integer(), + description="Resource tree root service name.", + default=colander.null, # if no parent + missing=colander.drop # if not returned (basic_info = True) + ) + root_service_type = colander.SchemaNode( + colander.Integer(), + description="Resource tree root service type.", + default=colander.null, # if no parent + missing=colander.drop # if not returned (basic_info = True) + ) + + +class ResourceTypes_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Get applicable children resource types under resource successful." + body = ResourceTypes_GET_ResponseBodySchema(code=HTTPOk.code, description=description) + + class ServiceResourcesBodySchema(ServiceBodySchema): children = ResourcesSchemaNode() @@ -1602,6 +1649,22 @@ class ServiceTypeResources_GET_RequestSchema(BaseRequestSchemaAPI): path = ServiceType_RequestPathSchema() +class ResourceTypesAllowed(colander.SequenceSchema): + description = "List of all allowed resource types at some level within the service hierarchy." + res_type_path = colander.SchemaNode( + colander.String(), + description="Allowed resource type under the service hierarchy." + ) + + +class ResourceStructuresAllowed(colander.SequenceSchema): + description = "List of allowed combinations of resource type hierarchical structures under the service." + res_type_path = colander.SchemaNode( + colander.String(), + description="Path-like resource type structure allowed under the service." + ) + + class ServiceTypeResourceInfo(colander.MappingSchema): resource_type = colander.SchemaNode( colander.String(), @@ -1609,8 +1672,10 @@ class ServiceTypeResourceInfo(colander.MappingSchema): ) resource_child_allowed = colander.SchemaNode( colander.Boolean(), - description="Indicates if the resource type allows child resources." + description="Indicates if the service allows any child resources." ) + resource_types_allowed = ResourceTypesAllowed() + resource_structure_allowed = ResourceStructuresAllowed() permission_names = PermissionNameListSchema( description="Permissions applicable to the specific resource type.", example=[Permission.READ.value, Permission.WRITE.value] @@ -3277,6 +3342,15 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema): "422": UnprocessableEntityResponseSchema(), "500": InternalServerErrorResponseSchema(), } +ResourceTypes_GET_responses = { + "200": ResourceTypes_GET_OkResponseSchema(), + "400": Resource_MatchDictCheck_BadRequestResponseSchema(), + "401": UnauthorizedResponseSchema(), + "403": Resource_MatchDictCheck_ForbiddenResponseSchema(), + "404": Resource_MatchDictCheck_NotFoundResponseSchema(), + "422": UnprocessableEntityResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} ServiceTypes_GET_responses = { "200": ServiceTypes_GET_OkResponseSchema(), "401": UnauthorizedResponseSchema(), diff --git a/magpie/services.py b/magpie/services.py index e13bb4138..2372e9ed5 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -1,5 +1,4 @@ import abc -import inspect import re from typing import TYPE_CHECKING @@ -61,6 +60,19 @@ def resource_type_names(cls): @property def child_resource_allowed(cls): # type: (Type[ServiceInterface]) -> bool + """ + Lists all resources allowed *somewhere* within its resource hierarchy under the service. + + .. note:: + Resources are not necessarily all allowed *directly* under the service. + This depends on whether :attr:`ServiceInterface.child_structure_allowed` is defined or not. + If not defined, resources are applicable anywhere. + Otherwise, they must respect the explicit structure definitions. + + .. seealso:: + Use :meth:`ServiceInterface.nested_resource_allowed` to obtain only scoped types allowed under a + given resource considering allowed path structures. + """ return len(cls.resource_types) > 0 @@ -68,12 +80,58 @@ def child_resource_allowed(cls): class ServiceInterface(object): # required service type identifier (unique) service_type = None # type: Str - # required request parameters for the service + """ + Service type identifier (required, unique across implementation). + """ + params_expected = [] # type: List[Str] - # global permissions allowed for the service (top-level resource) + """ + Request parameters that are expected and required for parsing service or child resource access. + """ + permissions = [] # type: List[Permission] - # dict of list for each corresponding allowed resource permissions (children resources) - resource_types_permissions = {} # type: Dict[models.Resource, List[Permission]] + """ + Permission allowed directly on the service as top-level resource. + """ + + resource_types_permissions = {} # type: Dict[Type[models.Resource], List[Permission]] + """ + Mapping of resource types to lists of permissions defining allowed children resource permissions under the service. + """ + + child_structure_allowed = [] # type: List[Str] + """ + Control listing of path-like resource types limiting the allowed structure of nested children resources. + + When not defined, any nested resource type combination is allowed if they themselves allow children resources. + Otherwise, nested child resource under the service can only be created at specific positions within the hierarchy + that matches exactly one of the listed control path-like definition. All definitions must start with ``service`` + and must contain at least one separator and a sub-resource to ensure working behaviour of child resource under + the service. + + For example, the below definition allows only resources typed ``route`` directly under the service. + The following nested resource under that first-level ``route`` can then be either another ``route`` followed + by a child ``process`` or directly a ``process``. Because ``process`` type doesn't allow any children resource + (see :attr:`models.Process.child_resource_allowed`), those are the only allowed combinations (cannot further nest + resources under the final ``process`` resource). Note that because intermediate ``route`` resources need to be + created at some point before the last ``process`` can even exist, partial paths (without ``process``) must also + be allowed as valid structures. + + .. code-block:: python + + child_structure_allowed = [ + "service/route", + "service/route/process", + "service/route/route", + "service/route/route/process", + ] + + .. seealso:: + - Validation of allowed nested children resource insertion of a given type under a parent resource is provided + by :meth:`ServiceInterface.validate_nested_resource_type` that employs :attr:`child_structure_allowed`. + - Listing of allowed resource types scoped under a given child resource within the hierarchy is provided + by :meth:`ServiceInterface.nested_resource_allowed`. + """ def __init__(self, service, request): # type: (models.Service, Request) -> None @@ -323,11 +381,80 @@ def get_resource_permissions(cls, resource_type_name): """ Obtains the allowed permissions of the service's child resource fetched by resource type name. """ - for res in cls.resource_types_permissions: # type: models.Resource + for res in cls.resource_types_permissions: # type: Type[models.Resource] if res.resource_type_name == resource_type_name: return cls.resource_types_permissions[res] return [] + @classmethod + def get_resource_type_path(cls, resource, extra_path=None): + # type: (ServiceOrResourceType, Optional[Str]) -> Str + """ + Generate the resource type path-like definition from the top service down to the specified resource. + + :param resource: leaf resource for which to generate the resource type path. + :param extra_path: optional resource path to append after the generated path. + :return: path like representation of the resource types from service to leaf resource path. + """ + session = get_db_session(obj=resource) + res_tree = reversed(list(models.RESOURCE_TREE_SERVICE.path_upper(resource.resource_id, db_session=session))) + res_extra_path = [] if extra_path is None else [extra_path] + res_types_path = "/".join([res.resource_type_name for res in res_tree] + res_extra_path) + return res_types_path + + @classmethod + def validate_nested_resource_type(cls, parent_resource, child_resource_type): + # type: (ServiceOrResourceType, Str) -> bool + """ + Validate whether a new child resource type is allowed under the parent resource under the service. + + :param parent_resource: Parent under which the new resource must be validated. This can be the service itself. + :param child_resource_type: Type to validate at the position defined under the parent resource. + :return: status indicating if insertion is allowed for this type and at this parent position. + """ + if not cls.child_resource_allowed: + return False + # make sure to obtain the specific resource/service implementation to avoid using the default + if parent_resource.resource_type_name == models.Service.resource_type_name: + res_impl = SERVICE_TYPE_DICT[parent_resource.type] + else: + res_impl = models.RESOURCE_TYPE_DICT[parent_resource.resource_type_name] + if not res_impl.child_resource_allowed: + return False + # if undefined control structures, assume any combination of nested resource is allowed (original behaviour) + if not cls.child_structure_allowed: + return True + res_types_path = cls.get_resource_type_path(parent_resource, extra_path=child_resource_type) + for allow_types_path in cls.child_structure_allowed: + if allow_types_path == res_types_path: + return True + return False + + @classmethod + def nested_resource_allowed(cls, parent_resource): + # type: (ServiceOrResourceType) -> List[Type[models.Resource]] + """ + Obtain the nested resource types allowed as children children resource within structure definitions. + """ + if not cls.child_resource_allowed: + return [] + # make sure to obtain the specific resource/service implementation to avoid using the default + if parent_resource.resource_type_name == models.Service.resource_type_name: + res_impl = SERVICE_TYPE_DICT[parent_resource.type] + else: + res_impl = models.RESOURCE_TYPE_DICT[parent_resource.resource_type_name] + if not res_impl.child_resource_allowed: + return [] + # if undefined control structures, any combination is allowed (original behaviour) + if not cls.child_structure_allowed: + return cls.resource_types + res_types_path = cls.get_resource_type_path(parent_resource, extra_path="") # terminate as ".../" + res_types_next = [ # retain only paths that correspond to immediately the next resource type + path.replace(res_types_path, "", 1) for path in cls.child_structure_allowed + if path.startswith(res_types_path) and len(path.replace(res_types_path, "", 1).split("/")) == 1 + ] + return [models.RESOURCE_TYPE_DICT[res_type] for res_type in res_types_next] + def allowed_permissions(self, resource): # type: (ServiceOrResourceType) -> List[Permission] """ @@ -849,10 +976,6 @@ class ServiceTHREDDS(ServiceInterface): Permission.WRITE, # NOTE: see special usage of WRITE in docs ] - params_expected = [ - "request" - ] - resource_types_permissions = { models.Directory: permissions, models.File: permissions, @@ -964,9 +1087,6 @@ class ServiceGeoserverMeta(ServiceMeta): "wfs": ServiceWFS, "wms": ServiceGeoserverWMS, "wps": ServiceWPS, - "WFS": ServiceWFS, - "WMS": ServiceGeoserverWMS, - "WPS": ServiceWPS, } @property @@ -1008,28 +1128,48 @@ class ServiceGeoserver(ServiceOWS): "service" ] + # only allow workspace directly under service + # then, only layer or process under that workspace + child_structure_allowed = [ + "{}/{}".format( + models.Service.resource_type_name, + models.Workspace.resource_type_name, + ), + "{}/{}/{}".format( + models.Service.resource_type_name, + models.Workspace.resource_type_name, + models.Layer.resource_type_name, + ), + "{}/{}/{}".format( + models.Service.resource_type_name, + models.Workspace.resource_type_name, + models.Process.resource_type_name, + ), + ] + def service_requested(self): # type: () -> Type[ServiceOWS] """ Obtain the applicable `OWS` implementation according to parsed request parameters. """ try: - svc = self.parser.params["service"] + svc_key = svc = self.parser.params["service"] except KeyError: - svc = None + svc_key = svc = None req = self.parser.params.get("request") - if svc is None and req is not None: + if svc_key is None and req is not None: # geoserver allows omitting 'service' request parameter because it can be inferred from the path # since all OWS services are accessed using '/geoserver/?request=...' # attempt to match using applicable 'request' parameter - for geo_svc in self.supported_ows: - if issubclass(geo_svc, ServiceInterface) and hasattr(geo_svc, "permissions"): - perm = Permission(req) - if perm in geo_svc.permissions: - svc = geo_svc + for svc_ows in self.supported_ows: + if issubclass(svc_ows, ServiceInterface) and hasattr(svc_ows, "permissions"): + perm = Permission(str(req).lower()) + if perm in svc_ows.permissions: + svc_key = svc_ows break + svc_key = str(svc_key).lower() ax.verify_param( - svc, is_in=True, param_compare=self.service_map, param_name="service", http_error=HTTPBadRequest, + svc_key, is_in=True, param_compare=self.service_map, param_name="service", http_error=HTTPBadRequest, content={ "service": self.service.resource_name, "type": self.service_type, @@ -1040,7 +1180,7 @@ def service_requested(self): "Unable to resolve the requested access for service: [{!s}].".format(self.service.resource_name) ) ) - return self.service_map[svc] + return self.service_map[svc_key] def resource_requested(self): # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index 93d89dae0..1cf688720 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -975,10 +975,10 @@ def add_resource(self): service_name=service_name, cur_svc_type=cur_svc_type)) - path = schemas.ServiceTypeResourceTypesAPI.path.format(service_type=cur_svc_type) + path = schemas.ResourceTypesAPI.path.format(resource_id=resource_id) resp = request_api(self.request, path, "GET") check_response(resp) - svc_res_types = get_json(resp)["resource_types"] + svc_res_types = get_json(resp)["children_resource_types"] data = { "service_name": service_name, "cur_svc_type": cur_svc_type, From 264890475cdb7a2a7da72f39f460f74e300ba173 Mon Sep 17 00:00:00 2001 From: fmigneault Date: Fri, 7 Jan 2022 19:07:17 -0500 Subject: [PATCH 08/34] fixes to reporting applied service configuration from API and setting one during creation from UI --- CHANGES.rst | 7 ++ .../api/management/service/service_formats.py | 10 +- magpie/api/schemas.py | 11 +- magpie/services.py | 93 +++++++++------- magpie/ui/home/static/style.css | 20 ++++ .../ui/management/templates/add_service.mako | 102 ++++++++++++------ .../ui/management/templates/edit_service.mako | 8 -- magpie/ui/management/views.py | 64 ++++++++--- magpie/utils.py | 12 +-- 9 files changed, 227 insertions(+), 100 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 58b9e5122..3d71e0fbb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -23,13 +23,20 @@ Features / Changes For backward compatibility, ``resource_types_allowed`` parameter already available in the same response will continue to report all possible ``Resource`` types *at any level* under the ``Service`` hierarchy, although not necessarily applicable as immediate child ``Resource`` under that ``Service``. +* Add ``configurable`` attribute to ``Service`` types that supports custom definitions modifying their behaviour. +* Add ``service_configurable`` to response of ``GET /service/{name}`` endpoint. * Adjust UI to consider ``child_structure_allowed`` definitions to propose only applicable ``Resource`` types in the combobox when creating a new ``Resource`` in the tree hierarchy. +* Add UI submission field to provide ``Service`` JSON configuration at creation when supported by the type. Bug Fixes ~~~~~~~~~~~~~~~~~~~~~ * Remove invalid ``request`` parameter in ``ServiceTHREDDS`` implementation. * Remove multiple invalid schema path definitions that are not mapped against any concrete API endpoint. +* Fix reporting of ``Service`` configuration for any type that supports it. Unless overridden during creation with a + custom configuration, ``ServiceTHREDDS`` implementation would not report their default configuration and would + instead return ``null``, making it difficult to know from the API if default or no configuration was being applied + for a given ``Service``. `3.20.0 `_ (2022-01-06) ------------------------------------------------------------------------------------ diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py index 6b0bcf271..ab6c9af96 100644 --- a/magpie/api/management/service/service_formats.py +++ b/magpie/api/management/service/service_formats.py @@ -65,6 +65,7 @@ def fmt_svc(): "service{}name".format(sep): str(service.resource_name), "service{}type".format(sep): str(service.type), "service{}sync_type".format(sep): svc_sync_type, + "service{}configurable".format(sep): SERVICE_TYPE_DICT[service.type].configurable, "resource{}id".format(sep): service.resource_id, } if show_public_url: @@ -74,9 +75,14 @@ def fmt_svc(): svc_info["service{}url".format(sep)] = str(service.url) if basic_info: return svc_info - if show_configuration: - svc_info["configuration"] = service.configuration svc_type = SERVICE_TYPE_DICT[service.type] + if show_configuration: + # make sure to generate the default configuration if applicable + if svc_type.configurable: + svc_config = svc_type(service, request=None).get_config() + else: + svc_config = None + svc_info["configuration"] = svc_config perms = svc_type.permissions if permissions is None else permissions svc_info.update(format_permissions(perms, permission_type)) if show_resources_allowed: diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index c9ce16e6d..8d3836c7e 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -890,7 +890,11 @@ class GroupDetailBodySchema(GroupPublicBodySchema, GroupInfoBodySchema): class ServiceConfigurationSchema(colander.MappingSchema): - description = "Custom configuration of the service. Expected format and fields specific to each service type." + description = ( + "Custom configuration of the service. " + "Expected format and fields specific to each service type. " + "Service type must support custom configuration." + ) missing = colander.drop default = colander.null @@ -915,6 +919,11 @@ class ServiceSummarySchema(colander.MappingSchema): description="Type of resource synchronization implementation.", example="thredds" ) + service_configurable = colander.SchemaNode( + colander.Boolean(), + description="Indicates if the service supports custom configuration.", + examble=False, + ) public_url = colander.SchemaNode( colander.String(), description="Proxy URL available for public access with permissions", diff --git a/magpie/services.py b/magpie/services.py index 2372e9ed5..042fa39c3 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -78,28 +78,27 @@ def child_resource_allowed(cls): @six.add_metaclass(ServiceMeta) class ServiceInterface(object): - # required service type identifier (unique) - service_type = None # type: Str + service_type = None # type: Optional[Str] # MUST be overridden """ Service type identifier (required, unique across implementation). """ - params_expected = [] # type: List[Str] + params_expected = [] # type: List[Str] """ Request parameters that are expected and required for parsing service or child resource access. """ - permissions = [] # type: List[Permission] + permissions = [] # type: List[Permission] """ Permission allowed directly on the service as top-level resource. """ - resource_types_permissions = {} # type: Dict[Type[models.Resource], List[Permission]] + resource_types_permissions = {} # type: Dict[Type[models.Resource], List[Permission]] """ Mapping of resource types to lists of permissions defining allowed children resource permissions under the service. """ - child_structure_allowed = [] # type: List[Str] + child_structure_allowed = [] # type: List[Str] """ Control listing of path-like resource types limiting the allowed structure of nested children resources. @@ -133,6 +132,12 @@ class ServiceInterface(object): by :meth:`ServiceInterface.nested_resource_allowed`. """ + _config = None # type: Optional[ServiceConfiguration] # for optimization to avoid reload and parsing each time + configurable = False + """ + Indicates if the service supports custom configuration. + """ + def __init__(self, service, request): # type: (models.Service, Request) -> None self.service = service # type: models.Service @@ -659,6 +664,8 @@ def _get_request(self): def _set_request(self, request): # type: (Request) -> None self._request = request + if request is None: + return # avoid error parsing undefined request # must reset the parser from scratch if request changes to ensure everything is updated with new inputs self.parser = ows_parser_factory(request) self.parser.parse(self.params_expected) # run parsing to obtain guaranteed lowered-name parameters @@ -958,7 +965,11 @@ class ServiceWFS(ServiceOWS): "typenames" ] - resource_types_permissions = {} + resource_types_permissions = { + # workspace must allow permissions for layers as well as parent in hierarchy + models.Workspace: models.Workspace.permissions + models.Layer.permissions, + models.Layer: models.Layer.permissions + } def resource_requested(self): return self.service, True # no children resource, so can only be the service @@ -981,15 +992,12 @@ class ServiceTHREDDS(ServiceInterface): models.File: permissions, } - def __init__(self, *_, **__): - super(ServiceTHREDDS, self).__init__(*_, **__) - self._config = None + configurable = True def get_config(self): # type: () -> ServiceConfiguration if self._config is not None: return self._config - self._config = super(ServiceTHREDDS, self).get_config() or {} self._config.setdefault("skip_prefix", "thredds") self._config.setdefault("file_patterns", [".*\\.nc"]) @@ -1147,49 +1155,60 @@ class ServiceGeoserver(ServiceOWS): ), ] + configurable = True + + def get_config(self): + # type: () -> ServiceConfiguration + """ + Obtain the configuration defining which `OWS` services are enabled under this instance. + + Should provide a mapping of all `OWS` service type names to enabled boolean status. + """ + if self._config is not None: + return self._config + self._config = super(ServiceGeoserver, self).get_config() or {} + for svc_type in type(self).service_map: + self._config.setdefault(svc_type, True) + if not isinstance(self._config[svc_type], bool): + self._config[svc_type] = True + self._config = {key: self._config[key] for key in sorted(self._config)} + return self._config + def service_requested(self): - # type: () -> Type[ServiceOWS] + # type: () -> Optional[Type[ServiceOWS]] """ Obtain the applicable `OWS` implementation according to parsed request parameters. """ - try: - svc_key = svc = self.parser.params["service"] - except KeyError: - svc_key = svc = None - req = self.parser.params.get("request") - if svc_key is None and req is not None: - # geoserver allows omitting 'service' request parameter because it can be inferred from the path + # guaranteed to exist and lowercase string if provided, otherwise None + svc = self.parser.params["service"] + req = self.parser.params["request"] + if not svc and req: + # geoserver allows omitting 'service' request query parameter because it can be inferred from the path # since all OWS services are accessed using '/geoserver/?request=...' # attempt to match using applicable 'request' parameter - for svc_ows in self.supported_ows: + for svc_ows in type(self).supported_ows: if issubclass(svc_ows, ServiceInterface) and hasattr(svc_ows, "permissions"): - perm = Permission(str(req).lower()) + perm = Permission(req) if perm in svc_ows.permissions: - svc_key = svc_ows + svc = svc_ows break - svc_key = str(svc_key).lower() - ax.verify_param( - svc_key, is_in=True, param_compare=self.service_map, param_name="service", http_error=HTTPBadRequest, - content={ - "service": self.service.resource_name, - "type": self.service_type, - "value": {"service": svc, "request": req} - }, - msg_on_fail=( - "Missing or unknown implementation inferred from OWS 'service' and 'request' parameters. " + - "Unable to resolve the requested access for service: [{!s}].".format(self.service.resource_name) - ) - ) - return self.service_map[svc_key] + config = self.get_config() + if svc not in config or not config[svc]: + return None + return type(self).service_map[svc] def resource_requested(self): # type: () -> Optional[Tuple[ServiceOrResourceType, bool]] svc = self.service_requested() + if not svc: + return None return svc(self.service, self.request).resource_requested() def permission_requested(self): - # type: () -> Permission + # type: () -> Optional[Union[Permission, Collection[Permission]]] svc = self.service_requested() + if not svc: + return None return svc(self.service, self.request).permission_requested() diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css index cece10d32..69fe61957 100644 --- a/magpie/ui/home/static/style.css +++ b/magpie/ui/home/static/style.css @@ -1250,6 +1250,10 @@ a.tab:visited { border-color: black; } +.input-configuration { + +} + .current-option { } @@ -1294,6 +1298,22 @@ table.fields-table input[type="radio"] { margin-left: 2em; } +.service-button { + margin-left: 40% +} + +.service-config-error { + margin-top: 0.25em; + margin-left: 0; + display: inline-flex; +} + +.service-config-error > img { + width: 1em; + height: 1em; + margin: 0.1em 0.25em 0.15em 0; +} + /* --- field form equal width inputs support different browsers using an higher level container diff --git a/magpie/ui/management/templates/add_service.mako b/magpie/ui/management/templates/add_service.mako index 95ea2a061..cd1d66403 100644 --- a/magpie/ui/management/templates/add_service.mako +++ b/magpie/ui/management/templates/add_service.mako @@ -9,14 +9,17 @@

Add Service

@@ -25,9 +28,12 @@ Service name: -
+
+ +
(unique) @@ -36,42 +42,78 @@ Service url: -
- - +
+ +
+ Service type:
- +
+ + + + + + + +
Configuration: +
+ +
+
+
+ (JSON) +
+ %if invalid_config: +
+ ERROR +
+ Invalid +
+
+ %endif
+ - - - +
Push to Phoenix: -
-
- +
+ +
+
+ +
diff --git a/magpie/ui/management/templates/edit_service.mako b/magpie/ui/management/templates/edit_service.mako index 07f1e7855..2ccb39a57 100644 --- a/magpie/ui/management/templates/edit_service.mako +++ b/magpie/ui/management/templates/edit_service.mako @@ -195,14 +195,6 @@
Configuration
-
- INFO - -
- This service employs the following custom configuration. -
-