Test

README.md

@@ -193,3 +193,4 @@ thread in our [Q&A forum][contact-qa], or send us an [email][contact-email].
 [res-ouath2]: <https://oauth.net/2/>
 [res-rabbitmq]: <https://www.rabbitmq.com/>
 [res-sem-ver]: <https://semver.org/>
+[res-docker compose]: <https://docs.docker.com/compose/>

pro_tes/app.py

@@ -16,6 +16,7 @@ def init_app() -> FlaskApp:
     """
     foca = Foca(
         config_file=Path(__file__).resolve().parent / "config.yaml",
+        custom_config_model="pro_tes.config_models.CustomConfig",
     )
     app = foca.create_app()
     with app.app.app_context():

pro_tes/config.yaml

@@ -106,44 +106,46 @@ exceptions:
   status_member: ["code"]
   exceptions: pro_tes.exceptions.exceptions
 
-controllers:
-  post_task:
-    db:
-      insert_attempts: 10
-    task_id:
-      charset: string.ascii_uppercase + string.digits
-      length: 6
-    timeout:
-      post: null
-      poll: 2
-      job: null
-    polling:
-      wait: 3
-      attempts: 100
-  list_tasks:
-    default_page_size: 5
+# Custom configuration
+custom:
+  controllers:
+    post_task:
+      db:
+        insert_attempts: 10
+      task_id:
+        charset: string.ascii_uppercase + string.digits
+        length: 6
+      timeout:
+        post: null
+        poll: 2
+        job: null
+      polling:
+        wait: 3
+        attempts: 100
+    list_tasks:
+      default_page_size: 5
   celery:
     monitor:
       timeout: 0.1
     message_maxsize: 16777216
 
-serviceInfo:
-  doc: Proxy TES for distributing tasks across a list of service TES instances
-  name: proTES
-  storage:
-    - file:///path/to/local/storage
+  serviceInfo:
+    doc: Proxy TES for distributing tasks across a list of service TES instances
+    name: proTES
+    storage:
+      - file:///path/to/local/storage
 
-tes:
-  service_list:
-    - "https://csc-tesk-noauth.rahtiapp.fi"
-    - "https://funnel.cloud.e-infra.cz/"
-    - "https://tesk-eu.hypatia-comp.athenarc.gr"
-    - "https://tesk-na.cloud.e-infra.cz"
-    - "https://vm4816.kaj.pouta.csc.fi/"
+  tes:
+    service_list:
+      - "https://csc-tesk-noauth.rahtiapp.fi"
+      - "https://funnel.cloud.e-infra.cz/"
+      - "https://tesk-eu.hypatia-comp.athenarc.gr"
+      - "https://tesk-na.cloud.e-infra.cz"
+      - "https://vm4816.kaj.pouta.csc.fi/"
 
-storeLogs:
-  execution_trace: True
+  storeLogs:
+    execution_trace: True
 
-middlewares:
-  - - "pro_tes.plugins.middlewares.task_distribution.distance.TaskDistributionDistance"
-    - "pro_tes.plugins.middlewares.task_distribution.random.TaskDistributionRandom"
+  middlewares:
+    - - "pro_tes.plugins.middlewares.task_distribution.distance.TaskDistributionDistance"
+      - "pro_tes.plugins.middlewares.task_distribution.random.TaskDistributionRandom"

pro_tes/config_models.py

@@ -0,0 +1,237 @@
+"""Custom app config models."""
+
+from typing import List, Optional
+import string
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+from pro_tes.ga4gh.tes.models import Service as TesServiceInfo
+
+
+# pragma pylint: disable=too-few-public-methods
+
+
+class DB(BaseModel):
+    """DB config for post_task.
+
+    Args:
+        insert_attempts: Number of attempts to insert a new task in DB.
+
+    Attributes:
+        insert_attempts: Number of attempts to insert a new task in DB.
+    """
+
+    insert_attempts: int = 10
+
+
+class TaskID(BaseModel):
+    """Task ID config.
+
+    Args:
+        charset: Characters to use when generating task IDs.
+        length: Length of the generated task ID.
+
+    Attributes:
+        charset: Characters to use when generating task IDs.
+        length: Length of the generated task ID.
+    """
+
+    charset: str = string.ascii_uppercase + string.digits
+    length: int = 6
+
+
+class Timeout(BaseModel):
+    """Timeout config.
+
+    Args:
+        post: Timeout for POST requests (None disables timeout).
+        poll: Timeout for polling.
+        job: Timeout for job execution (None disables timeout).
+
+    Attributes:
+        post: Timeout for POST requests (None disables timeout).
+        poll: Timeout for polling.
+        job: Timeout for job execution (None disables timeout).
+    """
+
+    post: Optional[int] = None
+    poll: int = 2
+    job: Optional[int] = None
+
+
+class Polling(BaseModel):
+    """Polling config.
+
+    Args:
+        wait: Wait time between polling attempts.
+        attempts: Max polling attempts before failure.
+
+    Attributes:
+        wait: Wait time between polling attempts.
+        attempts: Max polling attempts before failure.
+    """
+
+    wait: int = 3
+    attempts: int = 100
+
+
+class PostTask(BaseModel):
+    """Configuration for POST /task.
+
+    Args:
+        db: DB insert behavior.
+        task_id: Task ID generation settings.
+        timeout: Timeout settings.
+        polling: Polling behavior.
+
+    Attributes:
+        db: DB insert behavior.
+        task_id: Task ID generation settings.
+        timeout: Timeout settings.
+        polling: Polling behavior.
+    """
+
+    db: DB = DB()
+    task_id: TaskID = TaskID()
+    timeout: Timeout = Timeout()
+    polling: Polling = Polling()
+
+
+class ListTasks(BaseModel):
+    """Configuration for GET /tasks.
+
+    Args:
+        default_page_size: Default pagination size.
+
+    Attributes:
+        default_page_size: Default pagination size.
+    """
+
+    default_page_size: int = 5
+
+
+class Monitor(BaseModel):
+    """Celery monitor settings.
+
+    Args:
+        timeout: Timeout to wait for Celery monitoring.
+
+    Attributes:
+        timeout: Timeout to wait for Celery monitoring.
+    """
+
+    timeout: float = 0.1
+
+
+class Celery(BaseModel):
+    """Celery configuration.
+
+    Args:
+        monitor: Monitor settings.
+        message_maxsize: Maximum allowed message size.
+
+    Attributes:
+        monitor: Monitor settings.
+        message_maxsize: Maximum allowed message size.
+    """
+
+    monitor: Monitor = Monitor()
+    message_maxsize: int = 16777216
+
+
+class Controllers(BaseModel):
+    """Controller configurations.
+
+    Args:
+        post_task: Settings for POST /task.
+        list_tasks: Settings for GET /tasks.
+        celery: Celery background task settings.
+
+    Attributes:
+        post_task: Settings for POST /task.
+        list_tasks: Settings for GET /tasks.
+        celery: Celery background task settings.
+    """
+
+    post_task: PostTask = PostTask()
+    list_tasks: ListTasks = ListTasks()
+    celery: Celery = Celery()
+
+
+class Tes(BaseModel):
+    """TES backend configuration.
+
+    Args:
+        service_list: List of available TES services.
+
+    Attributes:
+        service_list: List of available TES services.
+    """
+
+    service_list: List[str] = [
+        "https://csc-tesk-noauth.rahtiapp.fi",
+        "https://funnel.cloud.e-infra.cz/",
+        "https://tesk-eu.hypatia-comp.athenarc.gr",
+        "https://tesk-na.cloud.e-infra.cz",
+        "https://vm4816.kaj.pouta.csc.fi/",
+    ]
+
+
+class StoreLogs(BaseModel):
+    """Logging configuration.
+
+    Args:
+        execution_trace: Whether to store execution trace logs.
+
+    Attributes:
+        execution_trace: Whether to store execution trace logs.
+    """
+
+    execution_trace: bool = True
+
+
+class Middlewares(BaseModel):
+    """Middleware configuration.
+
+    Args:
+        __root__: A list of middleware class paths.
+
+    Attributes:
+        __root__: A list of middleware class paths.
+    """
+
+    __root__: List[List[str]] = [
+        [
+            (
+                "pro_tes.plugins.middlewares.task_distribution.distance."
+                "TaskDistributionDistance"
+            ),
+            (
+                "pro_tes.plugins.middlewares.task_distribution.random."
+                "TaskDistributionRandom"
+            ),
+        ]
+    ]
+
+
+class CustomConfig(BaseModel):
+    """Custom app configuration.
+
+    Args:
+        controllers: All controller-related config.
+        tes: TES service list and defaults.
+        store_logs: Logging preferences.
+        middlewares: Middleware class paths.
+        service_info: Metadata about the service.
+
+    Attributes:
+        controllers: All controller-related config.
+        tes: TES service list and defaults.
+        store_logs: Logging preferences.
+        middlewares: Middleware class paths.
+        service_info: Metadata about the service.
+    """
+
+    controllers: Controllers = Controllers()
+    tes: Tes = Tes()
+    storeLogs: StoreLogs = StoreLogs()
+    middlewares: Middlewares = Middlewares()
+    service_info: TesServiceInfo

pro_tes/ga4gh/tes/service_info.py

@@ -65,7 +65,9 @@ def init_service_info_from_config(self) -> None:
 
         Set service info only if it does not yet exist.
         """
-        service_info_conf = current_app.config.foca.serviceInfo  # type: ignore
+        service_info_conf = (
+            current_app.config.foca.custom.serviceInfo  # type: ignore
+            )
         try:
             service_info_db = self.get_service_info()
         except NotFound:

pro_tes/ga4gh/tes/task_runs.py

@@ -62,7 +62,7 @@ def __init__(self) -> None:
         self.db_client: Collection = (
             self.foca_config.db.dbs["taskStore"].collections["tasks"].client
         )
-        self.store_logs = self.foca_config.storeLogs["execution_trace"]
+        self.store_logs = self.foca_config.custom.storeLogs["execution_trace"]
 
     def create_task(  # pylint: disable=too-many-statements,too-many-branches
         self, **kwargs
@@ -86,7 +86,7 @@ def create_task(  # pylint: disable=too-many-statements,too-many-branches
         # apply middlewares
         mw_handler = MiddlewareHandler()
         mw_handler.set_middlewares(
-            paths=current_app.config.foca.middlewares   # type: ignore
+            paths=current_app.config.foca.custom.middlewares   # type: ignore
         )
         logger.debug(f"Middlewares registered: {mw_handler.middlewares}")
         request_modified = mw_handler.apply_middlewares(request=request)
@@ -268,8 +268,10 @@ def list_tasks(self, **kwargs) -> dict:
         """
         page_size = kwargs.get(
             "page_size",
-            self.foca_config.controllers["list_tasks"]["default_page_size"],
-        )
+            self.foca_config.custom.controllers["list_tasks"][
+                "default_page_size"
+                ],
+            )
         page_token = kwargs.get("page_token")
         filter_dict = {}
 
@@ -427,7 +429,7 @@ def _write_doc_to_db(
         Returns:
             Tuple of task id and worker id.
         """
-        controller_config = self.foca_config.controllers["post_task"]
+        controller_config = self.foca_config.custom.controllers["post_task"]
         charset = controller_config["task_id"]["charset"]
         length = controller_config["task_id"]["length"]
 

pro_tes/plugins/middlewares/task_distribution/base.py

@@ -39,7 +39,9 @@ def apply_middleware(self, request: flask.Request) -> flask.Request:
             raise MiddlewareException("Request has no JSON payload.")
         self._set_tes_urls(
             tes_urls=deepcopy(
-                current_app.config.foca.tes["service_list"]  # type: ignore
+                current_app.config.foca.custom.tes[     # type: ignore
+                    "service_list"
+                    ]
             ),
             request=request,
         )