Merge pull request #309 from alexbrillant/step-cloner-apply-method

Use Apply Instead Of Recursive Methods

examples/hyperparams/plot_hyperparams.py

@@ -45,16 +45,16 @@ def main():
         ])
     ])
 
-    p.set_hyperparams_space(HyperparameterSpace({
+    p.set_hyperparams_space({
         'step1__multiply_by': RandInt(42, 50),
         'step2__multiply_by': RandInt(-10, 0),
         'Pipeline__PCA__n_components': RandInt(2, 3)
-    }))
+    })
 
     samples = p.get_hyperparams_space().rvs()
     p.set_hyperparams(samples)
 
-    samples = p.get_hyperparams()
+    samples = p.get_hyperparams().to_flat_as_dict_primitive()
     assert 42 <= samples['step1__multiply_by'] <= 50
     assert -10 <= samples['step2__multiply_by'] <= 0
     assert samples['Pipeline__PCA__n_components'] in [2, 3]

neuraxle/hyperparams/space.py

@@ -66,114 +66,187 @@
 
 from neuraxle.hyperparams.distributions import HyperparameterDistribution
 
-PARAMS_SPLIT_SEQ = "__"
 
-
-def nested_dict_to_flat(nested_hyperparams, dict_ctor=OrderedDict):
+class RecursiveDict(OrderedDict):
     """
-    Convert a nested hyperparameter dictionary to a flat one.
+    Wraps an hyperparameter nested dict or flat dict, and offer a few more functions.
+
+    This can be set on a Pipeline with the method ``set_hyperparams``.
 
-    :param nested_hyperparams: a nested hyperparameter dictionary.
-    :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict.
-    :return: a flat hyperparameter dictionary.
+    HyperparameterSamples are often the result of calling ``.rvs()`` on an HyperparameterSpace.
     """
-    ret = dict_ctor()
-    for k, v in nested_hyperparams.items():
-        if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor):
-            _ret = nested_dict_to_flat(v)
-            for key, val in _ret.items():
-                ret[k + PARAMS_SPLIT_SEQ + key] = val
+    DEFAULT_SEPARATOR = '__'
+
+    def __init__(self, *args, separator=None, **kwds):
+        if len(args) == 1 and isinstance(args[0], RecursiveDict) and len(kwds) == 0:
+            super().__init__(args[0].items())
+            separator = args[0].separator
         else:
-            ret[k] = v
-    return ret
+            super().__init__(*args, **kwds)
 
+        if separator is None:
+            separator = self.DEFAULT_SEPARATOR
 
-def flat_to_nested_dict(flat_hyperparams, dict_ctor=OrderedDict):
-    """
-    Convert a flat hyperparameter dictionary to a nested one.
+        self.separator = separator
 
-    :param flat_hyperparams: a flat hyperparameter dictionary.
-    :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict.
-    :return: a nested hyperparameter dictionary.
-    """
-    pre_ret = dict_ctor()
-    ret = dict_ctor()
-    for k, v in flat_hyperparams.items():
-        k, _, key = k.partition(PARAMS_SPLIT_SEQ)
-        if len(key) > 0:
-            if k not in pre_ret.keys():
-                pre_ret[k] = dict_ctor()
-            pre_ret[k][key] = v
-        else:
-            ret[k] = v
-    for k, v in pre_ret.items():
-        ret[k] = flat_to_nested_dict(v)
-    return ret
+    def __getitem__(self, item: str = None):
+        item_values = type(self)()
 
+        for name, values in self.items():
+            if item is None and not self.separator in name:
+                item_values[name] = values
 
-class HyperparameterSamples(OrderedDict):
-    """Wraps an hyperparameter nested dict or flat dict, and offer a few more functions.
+            if self.separator in name:
+                name_split = name.split(self.separator)
+                if str(name_split[0]) == item:
+                    item_values[self.separator.join(name_split[1:])] = values
 
-    This can be set on a Pipeline with the method ``set_hyperparams``.
+            if item == name:
+                item_values = values
 
-    HyperparameterSamples are often the result of calling ``.rvs()`` on an HyperparameterSpace."""
+        return item_values
 
-    def to_flat(self) -> 'HyperparameterSamples':
+    def to_flat(self) -> 'RecursiveDict':
         """
         Will create an equivalent flat HyperparameterSamples.
 
         :return: an HyperparameterSamples like self, flattened.
         """
-        return nested_dict_to_flat(self, dict_ctor=HyperparameterSamples)
+        return self.nested_dict_to_flat(dict_ctor=type(self))
 
-    def to_nested_dict(self) -> 'HyperparameterSamples':
+    def to_nested_dict(self) -> 'RecursiveDict':
         """
         Will create an equivalent nested dict HyperparameterSamples.
 
         :return: an HyperparameterSamples like self, as a nested dict.
         """
-        return flat_to_nested_dict(self, dict_ctor=HyperparameterSamples)
+        return self.flat_to_nested_dict(dict_ctor=type(self))
 
     def to_flat_as_dict_primitive(self) -> dict:
         """
         Will create an equivalent flat HyperparameterSpace, as a dict.
 
         :return: an HyperparameterSpace like self, flattened.
         """
-        return nested_dict_to_flat(self, dict_ctor=dict)
+        return self.nested_dict_to_flat(dict_ctor=dict)
 
     def to_nested_dict_as_dict_primitive(self) -> dict:
         """
         Will create an equivalent nested dict HyperparameterSpace, as a dict.
 
         :return: a nested primitive dict type of self.
         """
-        return flat_to_nested_dict(self, dict_ctor=dict)
+        return self.flat_to_nested_dict(dict_ctor=dict)
 
     def to_flat_as_ordered_dict_primitive(self) -> OrderedDict:
         """
         Will create an equivalent flat HyperparameterSpace, as a dict.
 
         :return: an HyperparameterSpace like self, flattened.
         """
-        return nested_dict_to_flat(self, dict_ctor=OrderedDict)
+        return self.nested_dict_to_flat(dict_ctor=OrderedDict)
 
     def to_nested_dict_as_ordered_dict_primitive(self) -> OrderedDict:
         """
         Will create an equivalent nested dict HyperparameterSpace, as a dict.
 
         :return: a nested primitive dict type of self.
         """
-        return flat_to_nested_dict(self, dict_ctor=OrderedDict)
+        return self.flat_to_nested_dict(dict_ctor=OrderedDict)
+
+    def nested_dict_to_flat(self, dict_ctor):
+        """
+        Convert a nested hyperparameter dictionary to a flat one.
 
+        :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict.
+        :return: a flat hyperparameter dictionary.
+        """
+        if isinstance(dict_ctor, RecursiveDict):
+            ret = dict_ctor(self.separator)
+        else:
+            ret = dict_ctor()
+
+        for k, v in self.items():
+            if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor):
+                nested_dict = v
+                if not isinstance(v, RecursiveDict):
+                    nested_dict = RecursiveDict(v, separator=self.separator)
+                _ret = nested_dict.nested_dict_to_flat(dict_ctor=type(nested_dict))
+                for key, val in _ret.items():
+                    ret[k + nested_dict.separator + key] = val
+            else:
+                ret[k] = v
+        return ret
+
+    def flat_to_nested_dict(self, dict_ctor=dict):
+        """
+        Convert a flat hyperparameter dictionary to a nested one.
+
+        :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict.
+        :return: a nested hyperparameter dictionary.
+        """
+        if isinstance(dict_ctor, RecursiveDict):
+            pre_ret = dict_ctor(self.separator)
+            ret = dict_ctor(self.separator)
+        else:
+            pre_ret = dict_ctor()
+            ret = dict_ctor()
+
+        for k, v in self.items():
+            k, _, key = k.partition(self.separator)
+            if len(key) > 0:
+                if k not in pre_ret.keys():
+                    pre_ret[k] = dict_ctor()
+                pre_ret[k][key] = v
+            else:
+                ret[k] = v
+        for k, v in pre_ret.items():
+            flat_dict = v
+            if not isinstance(v, RecursiveDict):
+                flat_dict = RecursiveDict(v, separator=self.separator)
+            ret[k] = flat_dict.flat_to_nested_dict(dict_ctor=dict_ctor)
+        return ret
+
+    def with_separator(self, separator):
+        """
+        Create a new recursive dict that uses the given separator at each level.
+
+        :param separator:
+        :return:
+        """
+        return type(self)(
+            separator=separator,
+            **{
+                key.replace(self.separator, separator): value if not isinstance(value, RecursiveDict) \
+                    else value.with_separator(separator) for key, value in self.items()
+            })
 
-class HyperparameterSpace(HyperparameterSamples):
-    """Wraps an hyperparameter nested dict or flat dict, and offer a few more functions to process
+
+class HyperparameterSamples(RecursiveDict):
+    """
+    Wraps an hyperparameter nested dict or flat dict, and offer a few more functions.
+
+    This can be set on a Pipeline with the method ``set_hyperparams``.
+
+    HyperparameterSamples are often the result of calling ``.rvs()`` on an HyperparameterSpace.
+    """
+
+    def __init__(self, *args, separator=None, **kwds):
+        super().__init__(*args, separator=separator, **kwds)
+
+
+class HyperparameterSpace(RecursiveDict):
+    """
+    Wraps an hyperparameter nested dict or flat dict, and offer a few more functions to process
     all contained HyperparameterDistribution.
 
     This can be set on a Pipeline with the method ``set_hyperparams_space``.
 
-    Calling ``.rvs()`` on an ``HyperparameterSpace`` results in ``HyperparameterSamples``."""
+    Calling ``.rvs()`` on an ``HyperparameterSpace`` results in ``HyperparameterSamples``.
+    """
+
+    def __init__(self, *args, separator=None, **kwds):
+        super().__init__(*args, separator=separator, **kwds)
 
     def rvs(self) -> 'HyperparameterSamples':
         """
@@ -196,7 +269,6 @@ def nullify(self):
             new_items.append((k, v))
         return HyperparameterSamples(new_items)
 
-
     def narrow_space_from_best_guess(
             self, best_guesses: 'HyperparameterSpace', kept_space_ratio: float = 0.5
     ) -> 'HyperparameterSpace':
@@ -228,19 +300,3 @@ def unnarrow(self) -> 'HyperparameterSpace':
                 v = v.unnarrow()
             new_items.append((k, v))
         return HyperparameterSpace(new_items)
-
-    def to_flat(self) -> 'HyperparameterSpace':
-        """
-        Will create an equivalent flat HyperparameterSpace.
-
-        :return: an HyperparameterSpace like self, flattened.
-        """
-        return nested_dict_to_flat(self, dict_ctor=HyperparameterSpace)
-
-    def to_nested_dict(self) -> 'HyperparameterSpace':
-        """
-        Will create an equivalent nested dict HyperparameterSpace.
-
-        :return: an HyperparameterSpace like self, as a nested dict.
-        """
-        return flat_to_nested_dict(self, dict_ctor=HyperparameterSpace)

neuraxle/metaopt/random.py

@@ -34,6 +34,7 @@
 from neuraxle.base import MetaStepMixin, BaseStep, ExecutionContext, HandleOnlyMixin, ForceHandleOnlyMixin, \
     EvaluableStepMixin
 from neuraxle.data_container import DataContainer
+from neuraxle.hyperparams.space import RecursiveDict
 from neuraxle.steps.loop import StepClonerForEachDataInput
 from neuraxle.steps.numpy import NumpyConcatenateOuterBatch, NumpyConcatenateOnCustomAxis
 
@@ -327,11 +328,13 @@ def disable_metrics(self):
         self.metrics_enabled = False
         if self.wrapped is not None:
             self.wrapped.apply('disable_metrics')
+        return RecursiveDict()
 
     def enable_metrics(self):
         self.metrics_enabled = True
         if self.wrapped is not None:
             self.wrapped.apply('enable_metrics')
+        return RecursiveDict()
 
     def _get_index_split(self, data_inputs):
         return math.floor(len(data_inputs) * (1 - self.test_size))

neuraxle/metrics.py

@@ -23,6 +23,7 @@
 
 from neuraxle.base import MetaStepMixin, BaseStep, ExecutionContext
 from neuraxle.data_container import DataContainer
+from neuraxle.hyperparams.space import RecursiveDict
 
 
 class MetricsWrapper(MetaStepMixin, BaseStep):
@@ -141,25 +142,26 @@ def _calculate_metrics_results(self, data_container: DataContainer):
         if self.print_metrics:
             self.print_fun(result)
 
-    def get_metrics(self) -> Dict:
+    def get_metrics(self) -> RecursiveDict:
         """
         Get all metrics results using the transformed data container, and the metrics function dict.
         To be used with :func:`neuraxle.base.BaseStep.apply` method.
 
         :return: dict with the step name as key, and all of the training, and validation metrics as values
         """
-        return {
+        return RecursiveDict({
             'train': self.metrics_results_train,
             'validation': self.metrics_results_validation
-        }
+        })
 
-    def toggle_metrics(self):
+    def toggle_metrics(self) -> RecursiveDict:
         """
         Toggle metrics wrapper on and off to temporarily disable metrics if needed..
 
         :return:
         """
         self.enabled = not self.enabled
+        return RecursiveDict({self.name: self.enabled})
 
     def disable_metrics(self):
         """
@@ -168,6 +170,7 @@ def disable_metrics(self):
         :return:
         """
         self.enabled = False
+        return RecursiveDict({self.name: self.enabled})
 
     def enable_metrics(self):
         """
@@ -176,3 +179,4 @@ def enable_metrics(self):
         :return:
         """
         self.enabled = True
+        return RecursiveDict({self.name: self.enabled})

neuraxle/steps/flow.py

@@ -295,12 +295,12 @@ def __init__(self, steps, hyperparams=None):
 
         if hyperparams is None:
             choices = list(self.keys())[:-1]
-            self.set_hyperparams(HyperparameterSamples({
+            self.set_hyperparams({
                 CHOICE_HYPERPARAM: choices[0]
-            }))
-            self.set_hyperparams_space(HyperparameterSpace({
+            })
+            self.set_hyperparams_space({
                 CHOICE_HYPERPARAM: Choice(choices)
-            }))
+            })
 
     def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, dict]):
         """