Add pydantic schema

Author: mvdbeek

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+per-file-ignores =
+    settings.py: E501

gravity/config_manager.py

@@ -12,14 +12,7 @@
 from yaml import safe_load
 
 from gravity import __version__
-from gravity.defaults import (
-    CELERY_DEFAULT_CONFIG,
-    DEFAULT_INSTANCE_NAME,
-    GUNICORN_DEFAULT_CONFIG,
-    GXIT_DEFAULT_IP,
-    GXIT_DEFAULT_PORT,
-    GXIT_DEFAULT_SESSIONS,
-)
+from gravity.settings import Settings
 from gravity.io import debug, error, exception, info, warn
 from gravity.state import (
     ConfigFile,
@@ -76,50 +69,38 @@ def __convert_config(self):
             os.unlink(config_state_json)
 
     def get_config(self, conf, defaults=None):
+        defaults = defaults or {}
         server_section = self.galaxy_server_config_section
         with open(conf) as config_fh:
             config_dict = safe_load(config_fh)
-
-        default_config = {
-            "galaxy_root": None,
-            "log_dir": join(expanduser(self.state_dir), "log"),
-            "virtualenv": None,
-            "instance_name": DEFAULT_INSTANCE_NAME,
-            "app_server": "gunicorn",
-            "gunicorn": GUNICORN_DEFAULT_CONFIG,
-            "celery": CELERY_DEFAULT_CONFIG,
-            "gx_it_proxy": {},
-            "handlers": {},
-        }
-        if defaults is not None:
-            recursive_update(default_config, defaults)
+        _gravity_config = config_dict.get(self.gravity_config_section) or {}
+        gravity_config = Settings(**recursive_update(defaults, _gravity_config))
+        if gravity_config.log_dir is None:
+            gravity_config.log_dir = join(expanduser(self.state_dir), "log")
 
         if server_section not in config_dict and self.gravity_config_section not in config_dict:
             error(f"Config file {conf} does not look like valid Galaxy, Reports or Tool Shed configuration file")
             return None
 
         app_config = config_dict.get(server_section) or {}
-        _gravity_config = config_dict.get(self.gravity_config_section) or {}
-        gravity_config = recursive_update(default_config, _gravity_config)
 
         config = ConfigFile()
         config.attribs = {}
         config.services = []
-        config.instance_name = gravity_config["instance_name"]
+        config.instance_name = gravity_config.instance_name
         config.config_type = server_section
-        config.attribs["app_server"] = gravity_config["app_server"]
-        config.attribs["log_dir"] = gravity_config["log_dir"]
-        config.attribs["virtualenv"] = gravity_config["virtualenv"]
-        config.attribs["gunicorn"] = gravity_config["gunicorn"]
-        config.attribs["celery"] = gravity_config["celery"]
-        config.attribs["handlers"] = gravity_config["handlers"]
-        config.attribs["gx_it_proxy"] = gravity_config["gx_it_proxy"]
+        config.attribs["app_server"] = gravity_config.app_server
+        config.attribs["log_dir"] = gravity_config.log_dir
+        config.attribs["virtualenv"] = gravity_config.virtualenv
+        config.attribs["gunicorn"] = gravity_config.gunicorn.dict()
+        config.attribs["celery"] = gravity_config.celery.dict()
+        config.attribs["handlers"] = gravity_config.handlers
         # Store gravity version, in case we need to convert old setting
         config.attribs['gravity_version'] = __version__
         webapp_service_names = []
 
         # shortcut for galaxy configs in the standard locations -- explicit arg ?
-        config.attribs["galaxy_root"] = app_config.get("root") or gravity_config.get("galaxy_root")
+        config.attribs["galaxy_root"] = app_config.get("root") or gravity_config.galaxy_root
         if config.attribs["galaxy_root"] is None:
             if os.environ.get("GALAXY_ROOT_DIR"):
                 config.attribs["galaxy_root"] = abspath(os.environ["GALAXY_ROOT_DIR"])
@@ -156,27 +137,26 @@ def get_config(self, conf, defaults=None):
         self.create_gxit_services(gravity_config, app_config, config)
         return config
 
-    def create_handler_services(self, gravity_config, config):
+    def create_handler_services(self, gravity_config: Settings, config):
         expanded_handlers = self.expand_handlers(gravity_config, config)
         for service_name, handler_settings in expanded_handlers.items():
             pools = handler_settings.get('pools')
             config.services.append(
                 service_for_service_type("standalone")(config_type=config.config_type, service_name=service_name, server_pools=pools))
 
-    def create_gxit_services(self, gravity_config, app_config, config):
-        if app_config.get("interactivetools_enable") and gravity_config["gx_it_proxy"].get("enable"):
+    def create_gxit_services(self, gravity_config: Settings, app_config, config):
+        if app_config.get("interactivetools_enable") and gravity_config.gx_it_proxy.enable:
             # TODO: resolve against data_dir, or bring in galaxy-config ?
             # CWD in supervisor template is galaxy_root, so this should work for simple cases as is
-            gxit_config = gravity_config['gx_it_proxy']
-            gxit_config["sessions"] = app_config.get("interactivetools_map", GXIT_DEFAULT_SESSIONS)
-            gxit_config["ip"] = gxit_config.get("ip", GXIT_DEFAULT_IP)
-            gxit_config["port"] = gxit_config.get("port", GXIT_DEFAULT_PORT)
-            gxit_config["verbose"] = '--verbose' if gxit_config.get("verbose") else ''
-            config.services.append(service_for_service_type("gx-it-proxy")(config_type=config.config_type, gxit=gxit_config))
+            gxit_config = gravity_config.gx_it_proxy
+            gxit_config.sessions = app_config.get("interactivetools_map", gxit_config.sessions)
+            gxit_config.verbose = '--verbose' if gxit_config.verbose else ''
+            config.services.append(service_for_service_type("gx-it-proxy")(config_type=config.config_type))
+        config.attribs["gx_it_proxy"] = gravity_config.gx_it_proxy.dict()
 
     @staticmethod
-    def expand_handlers(gravity_config, config):
-        handlers = gravity_config.get("handlers", {})
+    def expand_handlers(gravity_config: Settings, config):
+        handlers = gravity_config.handlers or {}
         expanded_handlers = {}
         default_name_template = "{name}_{process}"
         for service_name, handler_config in handlers.items():

gravity/defaults.py

@@ -0,0 +1,73 @@
+from enum import Enum
+from typing import (
+    Any,
+    Dict,
+    Optional,
+)
+from pydantic import (
+    BaseModel,
+    BaseSettings,
+    Field,
+)
+
+
+class LogLevel(str, Enum):
+    debug = "DEBUG"
+    info = "INFO"
+    warning = "WARNING"
+    error = "ERROR"
+
+
+class AppServer(str, Enum):
+    gunicorn = "gunicorn"
+    unicornherder = "unicornherder"
+
+
+class CelerySettings(BaseModel):
+
+    concurrency: int = Field(2, ge=0, description="Number of Celery Workers to start.")
+    loglevel: LogLevel = Field(LogLevel.debug, description="Log Level to use for Celery Worker.")
+    extra_args: str = Field(default="", description="Extra arguments to pass to Celery command line.")
+
+    class Config:
+        use_enum_values = True
+
+
+class GunicornSettings(BaseModel):
+    bind: str = Field(default="localhost:8080", description="The socket to bind. A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, ``fd://FD``. An IP is a valid HOST.")
+    workers: int = Field(default=1, ge=1, description=" Controls the number of Galaxy application processes Gunicorn will spawn. Increased web performance can be attained by increasing this value. If Gunicorn is the only application on the server, a good starting value is the number of CPUs * 2 + 1. 4-12 workers should be able to handle hundreds if not thousands of requests per second.")
+    timeout: int = Field(default=300, ge=0, description="Gunicorn workers silent for more than this many seconds are killed and restarted. Value is a positive number or 0. Setting it to 0 has the effect of infinite timeouts by disabling timeouts for all workers entirely. If you disable the ``preload`` option workers need to have finished booting within the timeout.")
+    extra_args: str = Field(default="", description="Extra arguments to pass to Gunicorn command line.")
+    preload: bool = Field(default=True, description="Use Gunicorn's --preload option to fork workers after loading the Galaxy Application. Consumes less memory when multiple processes are configured.")
+
+
+class GxItProxySettings(BaseModel):
+    enable: bool = Field(default=False, description="Set to true to start gx-it-proxy")
+    ip: str = Field(default="localhost", description="Public-facing IP of the proxy")
+    port: int = Field(default=4002, description="Public-facing port of the proxy")
+    sessions: str = Field(default="database/interactivetools_map.sqlite", description="Routes file to monitor. Should be set to the same path as ``interactivetools_map`` in the ``galaxy:`` section.")
+    verbose: bool = Field(default=True, description="Include verbose messages in gx-it-proxy")
+    forward_ip: Optional[str] = Field(default=None, description="Forward all requests to IP. This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network.")
+    forward_port: Optional[int] = Field(default=None, description="Forward all requests to port. This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network.")
+    reverse_proxy: Optional[bool] = Field(default=False, description="Cause the proxy to rewrite location blocks with its own port. This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network.")
+
+
+class Settings(BaseSettings):
+    """
+    Configuration for gravity process manager. Configure gravity in this section. ``uwsgi:`` section will be ignored if Galaxy is started via gravity commands (e.g ``./run.sh``, ``galaxy`` or ``galaxyctl``).
+    """
+    galaxy_root: Optional[str] = Field(None, description="Specify Galaxy's root directory. gravity will attempt to find the root directory, but you can set the directory explicitly with this option.")
+    log_dir: Optional[str] = Field(None, description="Set to a directory that should contain log files for the processes controlled by gravity. If not specified defaults to ``<state_dir>/logs``.")
+    virtualenv: Optional[str] = Field(None, description="Set to Galaxy's virtualenv directory. If not specified, gravity assumes all processes are on PATH.")
+    app_server: AppServer = Field(AppServer.gunicorn, description="Select the application server. ``gunicorn`` is the default application server. ``unicornherder`` is a production-oriented manager for (G)unicorn servers that automates zero-downtime Galaxy server restarts, similar to uWSGI Zerg Mode used in the past.")
+    instance_name: str = Field(default="_default_", description="Override the default instance name, this is hidden from you when running a single instance.")
+    gunicorn: GunicornSettings = Field(default={}, description="Configuration for Gunicorn.")
+    celery: CelerySettings = Field(default={}, description="Configuration for Celery Processes.")
+    gx_it_proxy: GxItProxySettings = Field(default={}, description="Configuration for gx-it-proxy.")
+    handlers: Dict[str, Dict[str, Any]] = Field(default={}, description="Configure dynamic handlers in this section. See https://docs.galaxyproject.org/en/latest/admin/scaling.html#dynamically-defined-handlers for details.")
+
+    class Config:
+        env_prefix = "gravity_"
+        env_nested_delimiter = "."
+        case_sensitive = False
+        use_enum_values = True

gravity/settings.py

@@ -36,7 +36,7 @@ def get_var(var_name):
     license="MIT",
     keywords="gravity galaxy",
     python_requires=">=3.6",
-    install_requires=["Click", "supervisor", "pyyaml", "ruamel.yaml"],
+    install_requires=["Click", "supervisor", "pyyaml", "ruamel.yaml", "pydantic"],
     entry_points={"console_scripts": [
         "galaxy = gravity.cli:galaxy",
         "galaxyctl = gravity.cli:galaxyctl",

setup.py

@@ -1,33 +1,27 @@
 import json
 from pathlib import Path
 
-from gravity.defaults import (
-    DEFAULT_GUNICORN_BIND,
-    DEFAULT_GUNICORN_TIMEOUT,
-    DEFAULT_GUNICORN_WORKERS,
-    DEFAULT_INSTANCE_NAME,
-    CELERY_DEFAULT_CONFIG
-
-)
+from gravity.settings import Settings
 
 
 def test_register_defaults(galaxy_yml, galaxy_root_dir, state_dir, default_config_manager):
     default_config_manager.add([str(galaxy_yml)])
     assert str(galaxy_yml) in default_config_manager.state['config_files']
     state = default_config_manager.state['config_files'][str(galaxy_yml)]
+    default_settings = Settings()
     assert state['config_type'] == 'galaxy'
-    assert state['instance_name'] == DEFAULT_INSTANCE_NAME
+    assert state['instance_name'] == default_settings.instance_name
     assert state['services'] == []
     attributes = state['attribs']
     assert attributes['app_server'] == 'gunicorn'
     assert Path(attributes['log_dir']) == Path(state_dir) / 'log'
     assert Path(attributes['galaxy_root']) == galaxy_root_dir
     gunicorn_attributes = attributes['gunicorn']
-    assert gunicorn_attributes['bind'] == DEFAULT_GUNICORN_BIND
-    assert gunicorn_attributes['workers'] == DEFAULT_GUNICORN_WORKERS
-    assert gunicorn_attributes['timeout'] == DEFAULT_GUNICORN_TIMEOUT
-    assert gunicorn_attributes['extra_args'] == ""
-    assert attributes['celery'] == CELERY_DEFAULT_CONFIG
+    assert gunicorn_attributes['bind'] == default_settings.gunicorn.bind
+    assert gunicorn_attributes['workers'] == default_settings.gunicorn.workers
+    assert gunicorn_attributes['timeout'] == default_settings.gunicorn.timeout
+    assert gunicorn_attributes['extra_args'] == default_settings.gunicorn.extra_args
+    assert attributes['celery'] == default_settings.celery.dict()
 
 
 def test_register_non_default(galaxy_yml, default_config_manager):
@@ -48,7 +42,8 @@ def test_register_non_default(galaxy_yml, default_config_manager):
     state = default_config_manager.state['config_files'][str(galaxy_yml)]
     gunicorn_attributes = state['attribs']['gunicorn']
     assert gunicorn_attributes['bind'] == new_bind
-    assert gunicorn_attributes['workers'] == DEFAULT_GUNICORN_WORKERS
+    default_settings = Settings()
+    assert gunicorn_attributes['workers'] == default_settings.gunicorn.workers
     celery_attributes = state['attribs']['celery']
     assert celery_attributes['concurrency'] == concurrency
 

tests/test_config_manager.py

@@ -42,8 +42,8 @@ def wait_for_gxit_proxy(state_dir):
     startup_logs = ""
     with open(state_dir / "log" / 'gx-it-proxy.log') as fh:
         for _ in range(STARTUP_TIMEOUT * 4):
-            startup_logs = fh.read()
-            if 'Listening' in startup_logs:
+            startup_logs = f"{startup_logs}{fh.read()}"
+            if 'Watching path' in startup_logs:
                 return True
             time.sleep(0.25)
     return startup_logs

tests/test_operations.py

@@ -0,0 +1,24 @@
+import json
+
+from gravity.settings import Settings
+
+
+def test_schema_json():
+    schema = Settings.schema_json(indent=2)
+    assert "Configuration for gravity process manager" in json.loads(schema)['description']
+
+
+def test_defaults_loaded():
+    settings = Settings()
+    assert settings.gunicorn.bind == 'localhost:8080'
+
+
+def test_defaults_override_constructor():
+    settings = Settings(**{'gunicorn': {'bind': 'localhost:8081'}})
+    assert settings.gunicorn.bind == 'localhost:8081'
+
+
+def test_defaults_override_env_var(monkeypatch):
+    monkeypatch.setenv("GRAVITY_GUNICORN.BIND", "localhost:8081")
+    settings = Settings()
+    assert settings.gunicorn.bind == 'localhost:8081'