Lazy/Eager evaluation of YAML tag nodes (#100)

* unittest for #99

* added config section and YAML tag for tests

* YAML constructors default to eager evaluation

* added test for lazy YAML evaluation

* YAML tag plugin marker

* documented yaml_tag

Author: maxfischer2781

cobald_tests/daemon/core/test_config.py

@@ -1,6 +1,7 @@
 from tempfile import NamedTemporaryFile
 
 import pytest
+import copy
 
 from cobald.daemon.config.mapping import ConfigurationError
 from cobald.daemon.core.config import load, COBalDLoader, yaml_constructor
@@ -13,6 +14,35 @@
 COBalDLoader.add_constructor(tag="!MockPool", constructor=yaml_constructor(MockPool))
 
 
+# Helpers for testing lazy/eager YAML evaluation
+# Since YAML defaults to lazy evaluation, the arguments available during evaluation
+# are not necessarily complete.
+class TagTracker:
+    """Helper to track the arguments supplied to YAML !Tags"""
+
+    def __init__(self, *args, **kwargs):
+        # the state of arguments *during* YAML evaluation
+        self.orig_args = copy.deepcopy(args)
+        self.orig_kwargs = copy.deepcopy(kwargs)
+        # the state of arguments *after* YAML evaluation
+        self.final_args = args
+        self.final_kwargs = kwargs
+
+
+COBalDLoader.add_constructor(
+    tag="!TagTrackerEager", constructor=yaml_constructor(TagTracker, eager=True)
+)
+COBalDLoader.add_constructor(
+    tag="!TagTrackerLazy", constructor=yaml_constructor(TagTracker, eager=False)
+)
+
+
+def get_config_section(config: dict, section: str):
+    return next(
+        content for plugin, content in config.items() if plugin.section == section
+    )
+
+
 class TestYamlConfig:
     def test_load(self):
         """Load a valid YAML config"""
@@ -95,10 +125,131 @@ def test_load_mixed_creation(self):
                     """
                 )
             with load(config.name) as config:
-                pipeline = next(
-                    content
-                    for plugin, content in config.items()
-                    if plugin.section == "pipeline"
-                )
+                pipeline = get_config_section(config, "pipeline")
                 assert isinstance(pipeline[0], LinearController)
                 assert isinstance(pipeline[0].target, MockPool)
+
+    def test_load_tags_substructure(self):
+        """Load !Tags with substructure"""
+        with NamedTemporaryFile(suffix=".yaml") as config:
+            with open(config.name, "w") as write_stream:
+                write_stream.write(
+                    """
+                    pipeline:
+                        - !MockPool
+                    __config_test:
+                        tagged: !TagTrackerEager
+                          host: 127.0.0.1
+                          port: 1234
+                          algorithm: HS256
+                          users:
+                            - user_name: tardis
+                              scopes:
+                                - user:read
+                    """
+                )
+            with load(config.name) as config:
+                tagged = get_config_section(config, "__config_test")["tagged"]
+                assert isinstance(tagged, TagTracker)
+                assert tagged.final_kwargs["host"] == "127.0.0.1"
+                assert tagged.final_kwargs["port"] == 1234
+                assert tagged.final_kwargs["algorithm"] == "HS256"
+                assert tagged.final_kwargs["users"][0]["user_name"] == "tardis"
+                assert tagged.final_kwargs["users"][0]["scopes"] == ["user:read"]
+
+    def test_load_tags_eager(self):
+        """Load !Tags with substructure, immediately using them"""
+        with NamedTemporaryFile(suffix=".yaml") as config:
+            with open(config.name, "w") as write_stream:
+                write_stream.write(
+                    """
+                    pipeline:
+                        - !MockPool
+                    __config_test:
+                        tagged: !TagTrackerEager
+                          top: "top level value"
+                          nested:
+                            - leaf: "leaf level value"
+                    """
+                )
+            with load(config.name) as config:
+                tagged = get_config_section(config, "__config_test")["tagged"]
+                assert isinstance(tagged, TagTracker)
+                # eager loading => all data should exist immediately
+                assert tagged.orig_kwargs["top"] == "top level value"
+                assert tagged.orig_kwargs["nested"] == [{"leaf": "leaf level value"}]
+                assert tagged.orig_kwargs == tagged.final_kwargs
+
+    def test_load_tags_lazy(self):
+        """Load !Tags with substructure, lazily using them"""
+        with NamedTemporaryFile(suffix=".yaml") as config:
+            with open(config.name, "w") as write_stream:
+                write_stream.write(
+                    """
+                    pipeline:
+                        - !MockPool
+                    __config_test:
+                        tagged: !TagTrackerLazy
+                          top: "top level value"
+                          nested:
+                            - leaf: "leaf level value"
+                    """
+                )
+            with load(config.name) as config:
+                tagged = get_config_section(config, "__config_test")["tagged"]
+                assert isinstance(tagged, TagTracker)
+                # eager loading => only some data should exist immediately...
+                assert tagged.orig_kwargs["top"] == "top level value"
+                assert tagged.orig_kwargs["nested"] == []
+                # ...but should be there in the end
+                assert tagged.final_kwargs["nested"] == [{"leaf": "leaf level value"}]
+
+    def test_load_tags_nested(self):
+        """Load !Tags with nested !Tags"""
+        with NamedTemporaryFile(suffix=".yaml") as config:
+            with open(config.name, "w") as write_stream:
+                write_stream.write(
+                    """
+                    pipeline:
+                        - !MockPool
+                    __config_test:
+                        top_eager: !TagTrackerEager
+                          nested:
+                          - leaf: "leaf level value"
+                          - leaf_lazy: !TagTrackerLazy
+                              nested:
+                                - leaf: "leaf level value"
+                    """
+                )
+            with load(config.name) as config:
+                top_eager = get_config_section(config, "__config_test")["top_eager"]
+                # eager tags are evaluated eagerly
+                assert top_eager.orig_kwargs["nested"][0] == {
+                    "leaf": "leaf level value"
+                }
+                leaf_lazy = top_eager.orig_kwargs["nested"][1]["leaf_lazy"]
+                # eagerness overrides laziness
+                assert leaf_lazy.orig_kwargs["nested"] == [{"leaf": "leaf level value"}]
+
+    def test_load_tag_settings(self):
+        """Load !Tags with decorator settings"""
+        # __yaml_tag_test is provided by the cobald package
+        with NamedTemporaryFile(suffix=".yaml") as config:
+            with open(config.name, "w") as write_stream:
+                write_stream.write(
+                    """
+                    pipeline:
+                        - !MockPool
+                    __config_test:
+                        settings_tag: !__yaml_tag_test
+                            top: "top level value"
+                            nested:
+                            - leaf: "leaf level value"
+                    """
+                )
+            with load(config.name) as config:
+                section = get_config_section(config, "__config_test")
+                args, kwargs = section["settings_tag"]
+                assert args == ()
+                assert kwargs["top"] == "top level value"
+                assert kwargs["nested"] == [{"leaf": "leaf level value"}]

docs/source/changes/100.yaml_tag_settings.yaml

@@ -0,0 +1,11 @@
+category: added
+summary: "YAML ``!tags`` may be eagerly evaluated"
+description: |
+  The YAML configuration is usually evaluated lazily, as YAML documents can be
+  recursive. YAML ``!tag`` plugins that require their part of the configuration
+  promptly can now be decorated with `:py:func:`cobald.daemon.plugins.yaml_tag`
+  to enforce eager evaluation.
+issues:
+- 99
+pull requests:
+- 100

docs/source/custom/package.rst

@@ -157,6 +157,15 @@ can be made available as ``MyExtension`` in this way:
     If a plugin implements a :py:meth:`~cobald.interfaces.Controller.s` method,
     this is used automatically.
 
+.. note::
+
+    If a plugin requires eager loading of its YAML configuration,
+    decorate it with :py:func:`cobald.daemon.plugins.yaml_tag`.
+
+.. versionadded:: 0.12.3
+
+    The :py:func:`cobald.daemon.plugins.yaml_tag` and eager evaluation.
+
 Section Plugins
 ---------------
 

setup.py

@@ -45,10 +45,12 @@
                     ("Limiter", "cobald.decorator.limiter"),
                     ("Logger", "cobald.decorator.logger"),
                     ("Standardiser", "cobald.decorator.standardiser"),
+                    ("__yaml_tag_test", "cobald.daemon.plugins"),
                 )
             ],
             "cobald.config.sections": [
-                "pipeline = cobald.daemon.core.config:load_pipeline"
+                "pipeline = cobald.daemon.core.config:load_pipeline",
+                "__config_test = builtins:dict",
             ],
         },
         # >>> Dependencies

src/cobald/daemon/config/yaml.py

@@ -1,4 +1,4 @@
-from typing import Type, Tuple
+from typing import Type, Tuple, Callable, TypeVar
 
 from yaml import SafeLoader, BaseLoader, nodes
 
@@ -9,6 +9,9 @@
 )
 
 
+R = TypeVar("R")
+
+
 def load_configuration(
     path: str, loader: Type[BaseLoader] = SafeLoader, plugins: Tuple[SectionPlugin] = ()
 ):
@@ -21,18 +24,20 @@ def load_configuration(
     return load_mapping_configuration(config_data=config_data, plugins=plugins)
 
 
-def yaml_constructor(factory):
+def yaml_constructor(
+    factory: Callable[..., R], *, eager: bool = False
+) -> Callable[..., R]:
     """
     Convert a factory function/class to a YAML constructor
 
     :param factory: the factory function/class
+    :param eager: whether the YAML must be evaluated eagerly
     :return: factory constructor
 
     Applying this helper to a factory allows it to be used as a YAML constructor,
     without it knowing about YAML itself.
     It properly constructs nodes and converts
-    mapping nodes to ``factory(**node)``,
-    sequence nodes to ``factory(*node)``, and
+    mapping nodes to ``factory(**node)``, sequence nodes to ``factory(*node)``, and
     scalar nodes to ``factory()``.
 
     For example, registering the constructor ``yaml_constructor(factory)`` as
@@ -43,16 +48,20 @@ def yaml_constructor(factory):
         - !factory
           a: 0.3
           b: 0.7
+
+    Since YAML can express recursive data, nested data structures are evaluated lazily
+    by default. Set ``eager=True`` to enforce eager evaluation before calling the
+    constructor.
     """
 
     def factory_constructor(loader: BaseLoader, node: nodes.Node):
         if isinstance(node, nodes.MappingNode):
-            kwargs = loader.construct_mapping(node)
+            kwargs = loader.construct_mapping(node, deep=eager)
             return factory(**kwargs)
         elif isinstance(node, nodes.ScalarNode):
             return factory()
         elif isinstance(node, nodes.SequenceNode):
-            args = loader.construct_sequence(node)
+            args = loader.construct_sequence(node, deep=eager)
             return factory(*args)
         else:
             raise ConfigurationError(

src/cobald/daemon/core/config.py

@@ -6,7 +6,7 @@
 from entrypoints import get_group_all as get_entrypoints
 from toposort import toposort_flatten
 
-from ..plugins import constraints as plugin_constraints
+from ..plugins import constraints as plugin_constraints, YAMLTagSettings
 from ..config.yaml import (
     load_configuration as load_yaml_configuration,
     yaml_constructor,
@@ -42,8 +42,10 @@ def add_constructor_plugins(entry_point_group: str, loader: Type[BaseLoader]) ->
             pipeline_factory = entry.load().s
         except AttributeError:
             pipeline_factory = entry.load()
+        settings = YAMLTagSettings.fetch(pipeline_factory)
         loader.add_constructor(
-            tag="!" + entry.name, constructor=yaml_constructor(pipeline_factory)
+            tag="!" + entry.name,
+            constructor=yaml_constructor(pipeline_factory, eager=settings.eager),
         )
 
 

src/cobald/daemon/plugins.py

@@ -1,7 +1,7 @@
 """
 Tools and helpers to declare plugins
 """
-from typing import Iterable, FrozenSet, TypeVar
+from typing import Iterable, FrozenSet, TypeVar, NamedTuple
 
 
 T = TypeVar("T")
@@ -54,3 +54,54 @@ def section_wrapper(plugin: T) -> T:
         return plugin
 
     return section_wrapper
+
+
+class YAMLTagSettings(NamedTuple):
+    """Settings for interpreting a YAML tag"""
+
+    eager: bool = False
+
+    @classmethod
+    def fetch(cls, plugin):
+        """Provide the settings for `plugin`"""
+        try:
+            return plugin.__cobald_yaml_tag__
+        except AttributeError:
+            return cls()
+
+    def mark(self, plugin):
+        """Mark `plugin` to use the current settings"""
+        plugin.__cobald_yaml_tag__ = self
+
+
+def yaml_tag(*, eager: bool = False):
+    """
+    Mark a callable as a YAML tag constructor with specific settings
+
+    :param eager: whether the YAML content must be evaluated eagerly
+
+    Since YAML can express recursive data, nested data structures are evaluated lazily
+    by default. This means a constructor receives nested data structures
+    (e.g. a ``dict`` of ``dict``s) upfront but nested content is added later on.
+    If a constructor requires the entire data at once, set ``eager=True`` to enforce
+    eager evaluation before calling the constructor.
+
+    .. note::
+
+        This decorator only serves to apply non-default settings for a plugin.
+        A plugin must still be registered using ``entry_points``.
+    """
+
+    def mark_settings(plugin: T) -> T:
+        YAMLTagSettings(eager=eager).mark(plugin)
+        return plugin
+
+    return mark_settings
+
+
+@yaml_tag(eager=True)
+def __yaml_tag_test(*args, **kwargs):
+    """YAML tag constructor for testing only"""
+    import copy
+
+    return copy.deepcopy(args), copy.deepcopy(kwargs)