From 7ac675a7162629d9bab7a103f6d6c6c18330b954 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 4 Apr 2020 16:54:12 -0400 Subject: [PATCH 01/48] Wip Use Apply Instead Of Multiple Recursive Methods --- neuraxle/base.py | 234 ++++++------------ neuraxle/steps/loop.py | 53 ++-- .../test_step_cloner_for_each_data_input.py | 4 + 3 files changed, 99 insertions(+), 192 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index dee0f17f..56f66435 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -41,6 +41,8 @@ from neuraxle.data_container import DataContainer from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples +RECURSIVE_STEP_SEPARATOR = '__' + DEFAULT_CACHE_FOLDER = os.path.join(os.getcwd(), 'cache') @@ -667,6 +669,7 @@ def set_train(self, is_train: bool = True): :func:`BaseStep.set_train` """ self.is_train = is_train + self.apply(method_name='set_train', is_train=is_train, children_only=True) return self def set_name(self, name: str): @@ -735,7 +738,18 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ self.invalidate() - self.hyperparams = HyperparameterSamples(hyperparams).to_flat() + + remainders = dict() + for name, hparams in hyperparams.items(): + if RECURSIVE_STEP_SEPARATOR in name: + if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): + self.hyperparams = HyperparameterSamples(hyperparams[hparams]) + else: + remainders[name] = hparams + else: + self.hyperparams = HyperparameterSamples(hyperparams[hparams]) + + self.apply(method_name='set_hyperparams', hyperparams=remainders, children_only=True) return self def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': @@ -768,6 +782,18 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': """ self.hyperparams.update(hyperparams) self.hyperparams = HyperparameterSamples(self.hyperparams).to_flat() + + remainders = dict() + for name, hparams in hyperparams.items(): + if RECURSIVE_STEP_SEPARATOR in name: + if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): + self.hyperparams.update(HyperparameterSamples(hparams).to_flat()) + else: + remainders[name] = hparams + else: + self.hyperparams.update(HyperparameterSamples(hparams).to_flat()) + + self.apply(method_name='update_hyperparams', hyperparams=remainders, children_only=True) return self def get_hyperparams(self) -> HyperparameterSamples: @@ -839,7 +865,18 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ self.invalidate() - self.hyperparams_space = HyperparameterSpace(hyperparams_space).to_flat() + + remainders = dict() + for name, hparams_space in hyperparams_space.items(): + if RECURSIVE_STEP_SEPARATOR in name: + if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): + self.hyperparams_space = HyperparameterSpace(hparams_space).to_flat() + else: + remainders[name] = hparams_space + else: + self.hyperparams_space = HyperparameterSpace(hyperparams_space).to_flat() + + self.apply(method_name='set_hyperparams_space', hyperparams_space=remainders, children_only=True) return self def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': @@ -872,6 +909,18 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B """ self.hyperparams_space.update(hyperparams_space) self.hyperparams_space = HyperparameterSamples(self.hyperparams_space).to_flat() + + remainders = dict() + for name, hparams_space in hyperparams_space.items(): + if RECURSIVE_STEP_SEPARATOR in name: + if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): + self.hyperparams_space.update(HyperparameterSpace(hparams_space).to_flat()) + else: + remainders[name] = hparams_space + else: + self.hyperparams_space.update(HyperparameterSpace(hparams_space).to_flat()) + + self.apply(method_name='update_hyperparams_space', hyperparams_space=remainders, children_only=True) return self def get_hyperparams_space(self) -> HyperparameterSpace: @@ -918,16 +967,20 @@ def _inverse_transform_data_container( return data_container - def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Dict: + def apply_method(self, method: Callable, step_name=None, children_only: bool=False, *kargs, **kwargs) -> Dict: """ Apply a method to a step and its children. :param method: method to call with self :param step_name: current pipeline step name + :param children_only: apply method only to the sub steps :param kargs: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ + if children_only: + return {} + if step_name is not None: step_name = "{}__{}".format(step_name, self.name) else: @@ -937,17 +990,20 @@ def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Di step_name: method(self, *kargs, **kwargs) } - def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + def apply(self, method_name: str, step_name=None, children_only: bool=False, *kargs, **kwargs) -> Dict: """ Apply a method to a step and its children. :param method_name: method name that need to be called on all steps :param step_name: current pipeline step name + :param children_only: apply method only to the sub steps :param kargs: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ results = {} + if children_only: + return results if step_name is not None: step_name = "{}__{}".format(step_name, self.name) @@ -1308,8 +1364,8 @@ def _invalidate(step): if full_dump: # initialize and invalidate steps to make sure that all steps will be saved - self.apply_method(_initialize_if_needed) - self.apply_method(_invalidate) + self.apply_method(method=_initialize_if_needed) + self.apply_method(method=_invalidate) context.mkdir() stripped_step = copy(self) @@ -1658,33 +1714,6 @@ def teardown(self) -> BaseStep: self.is_initialized = False return self - def set_train(self, is_train: bool = True): - """ - Set pipeline step mode to train or test. Also set wrapped step mode to train or test. - - For instance, you can add a simple if statement to direct to the right implementation: - - .. code-block:: python - - def transform(self, data_inputs): - if self.is_train: - self.transform_train_(data_inputs) - else: - self.transform_test_(data_inputs) - - def fit_transform(self, data_inputs, expected_outputs=None): - if self.is_train: - self.fit_transform_train_(data_inputs, expected_outputs) - else: - self.fit_transform_test_(data_inputs, expected_outputs) - - :param is_train: bool - :return: - """ - self.is_train = is_train - self.wrapped.set_train(is_train) - return self - def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: """ Set step hyperparameters, and wrapped step hyperparams with the given hyperparams. @@ -1910,17 +1939,18 @@ def resume(self, data_container: DataContainer, context: ExecutionContext): data_container = self._did_process(data_container, context) return data_container - def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + def apply(self, method_name: str, step_name=None, children_only=False, *kargs, **kwargs) -> Dict: """ Apply the method name to the meta step and its wrapped step. :param method_name: method name that need to be called on all steps :param step_name: step name to apply the method to + :param children_only: apply method to children only :param kargs: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = BaseStep.apply(self, method_name=method_name, step_name=step_name, *kargs, **kwargs) + results = BaseStep.apply(self, method_name=method_name, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: step_name = "{}__{}".format(step_name, self.name) @@ -1933,17 +1963,18 @@ def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: return results - def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Union[Dict, Iterable]: + def apply_method(self, method: Callable, step_name=None, children_only=False, *kargs, **kwargs) -> Union[Dict, Iterable]: """ Apply method to the meta step and its wrapped step. :param method: method to call with self :param step_name: step name to apply the method to + :param children_only: apply method to children only :param kargs: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = BaseStep.apply_method(self, method=method, step_name=step_name, *kargs, **kwargs) + results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: step_name = "{}__{}".format(step_name, self.name) @@ -2302,17 +2333,18 @@ def teardown(self) -> 'BaseStep': return self - def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + def apply(self, method_name: str, step_name=None, children_only: bool = False, *kargs, **kwargs) -> Dict: """ Apply the method name to the pipeline step and all of its children. :param method_name: method name that need to be called on all steps :param step_name: current pipeline step name + :param children_only: apply method only to the sub steps :param kargs: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = BaseStep.apply(self, method_name, step_name=step_name, *kargs, **kwargs) + results = BaseStep.apply(self, method_name, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: step_name = "{}__{}".format(step_name, self.name) @@ -2325,17 +2357,18 @@ def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: return results - def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Dict: + def apply_method(self, method: Callable, step_name=None, children_only=False, *kargs, **kwargs) -> Dict: """ Apply a method to the pipeline step and all of its children. :param method: method to call with self :param step_name: current pipeline step name + :param children_only: apply method only to the sub steps :param kargs: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = BaseStep.apply_method(self, method=method, step_name=step_name, *kargs, **kwargs) + results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: step_name = "{}__{}".format(step_name, self.name) @@ -2477,65 +2510,6 @@ def get_hyperparams(self) -> HyperparameterSamples: return hyperparams.to_flat() - def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, OrderedDict, dict]) -> BaseStep: - """ - Set step hyperparameters to the given :class:`~neuraxle.space.HyperparameterSamples`. - - Example : - - .. code-block:: python - - p = Pipeline([SomeStep()]) - p.set_hyperparams(HyperparameterSamples({ - 'learning_rate': 0.1, - 'some_step__learning_rate': 0.2 # will set SomeStep() hyperparam 'learning_rate' to 0.2 - })) - - :return: step hyperparameters - - .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - self.invalidate() - - hyperparams: HyperparameterSamples = HyperparameterSamples(hyperparams).to_nested_dict() - - remainders = dict() - for name, hparams in hyperparams.items(): - if name in self.steps.keys(): - self.steps[name].set_hyperparams(HyperparameterSamples(hparams)) - else: - remainders[name] = hparams - self.hyperparams = HyperparameterSamples(remainders) - - return self - - def update_hyperparams(self, hyperparams: Union[HyperparameterSamples, OrderedDict, dict]) -> BaseStep: - """ - Update the steps hyperparameters without removing the already-set hyperparameters. - Please refer to :func:`~BaseStep.update_hyperparams`. - - :param hyperparams: hyperparams to update - :return: step - - .. seealso:: - :func:`~BaseStep.update_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - self.invalidate() - - hyperparams: HyperparameterSamples = HyperparameterSamples(hyperparams).to_nested_dict() - - remainders = dict() - for name, hparams in hyperparams.items(): - if name in self.steps.keys(): - self.steps[name].update_hyperparams(HyperparameterSamples(hparams)) - else: - remainders[name] = hparams - self.hyperparams.update(remainders) - - return self - def get_hyperparams_space(self): """ Get step hyperparameters space as :class:`~neuraxle.space.HyperparameterSpace`. @@ -2570,66 +2544,6 @@ def get_hyperparams_space(self): return all_hyperparams.to_flat() - def update_hyperparams_space(self, hyperparams_space: Union[HyperparameterSpace, OrderedDict, dict]) -> BaseStep: - """ - Update the steps hyperparameters without removing the already-set hyperparameters. - Please refer to :func:`~BaseStep.update_hyperparams`. - - :param hyperparams_space: hyperparams_space to update - :return: step - - .. seealso:: - :func:`~BaseStep.update_hyperparams`, - :class:`~neuraxle.space.HyperparameterSamples` - """ - self.is_invalidated = True - - hyperparams_space: HyperparameterSpace = HyperparameterSpace(hyperparams_space).to_nested_dict() - - remainders = dict() - for name, hparams_space in hyperparams_space.items(): - if name in self.steps.keys(): - self.steps[name].update_hyperparams_space(HyperparameterSamples(hparams_space)) - else: - remainders[name] = hparams_space - self.hyperparams_space.update(remainders) - - return self - - def set_hyperparams_space(self, hyperparams_space: Union[HyperparameterSpace, OrderedDict, dict]) -> BaseStep: - """ - Set step hyperparameters space as :class:`~neuraxle.hyperparams.space.HyperparameterSpace`. - - Example : - - .. code-block:: python - - p = Pipeline([SomeStep()]) - p.set_hyperparams_space(HyperparameterSpace({ - 'learning_rate': RandInt(0,5), - 'some_step__learning_rate': RandInt(0, 10) # will set SomeStep() 'learning_rate' hyperparam space to RandInt(0, 10) - })) - - :param hyperparams_space: hyperparameters space - :return: self - - .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSpace` - """ - self.invalidate() - - hyperparams_space: HyperparameterSpace = HyperparameterSpace(hyperparams_space).to_nested_dict() - - remainders = dict() - for name, hparams in hyperparams_space.items(): - if name in self.keys(): - self.steps[name].set_hyperparams_space(HyperparameterSpace(hparams)) - else: - remainders[name] = hparams - self.hyperparams_space = HyperparameterSpace(remainders) - - return self - def should_save(self): """ Returns if the step needs to be saved or not. diff --git a/neuraxle/steps/loop.py b/neuraxle/steps/loop.py index adf2df09..711e5e3a 100644 --- a/neuraxle/steps/loop.py +++ b/neuraxle/steps/loop.py @@ -23,7 +23,7 @@ """ import copy -from typing import List +from typing import List, Callable, Dict, Iterable, Union import numpy as np @@ -157,38 +157,19 @@ def __init__(self, wrapped: BaseStep, copy_op=copy.deepcopy, cache_folder_when_n self.steps_as_tuple: List[NamedTupleList] = [] self.copy_op = copy_op - def set_train(self, is_train: bool = True): - MetaStepMixin.set_train(self, is_train) - [step.set_train(is_train) for _, step in self] - return self - - def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: - MetaStepMixin.set_hyperparams(self, hyperparams) - self.steps_as_tuple = [(name, step.set_hyperparams(self.wrapped.get_hyperparams())) for name, step in self] - return self - - def update_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: - """ - Update the step hyperparameters without removing the already-set hyperparameters. - Please refer to :func:`~BaseStep.update_hyperparams`. + def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + results = MetaStepMixin.apply(self, method_name=method_name, step_name=step_name, *kargs, **kwargs) + if len(self) > 0: + for _, step in self: + results.update(step.apply(method_name=method_name, step_name=step_name, *kargs, **kwargs)) + return results - :param hyperparams: hyperparams to update - :type hyperparams: HyperparameterSamples - :return: self - :rtype: BaseStep - - .. seealso:: - :func:`~BaseStep.update_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - MetaStepMixin.update_hyperparams(self, hyperparams) - self.steps_as_tuple = [(name, step.set_hyperparams(self.wrapped.get_hyperparams())) for name, step in self.steps_as_tuple] - return self - - def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': - MetaStepMixin.set_hyperparams_space(self, hyperparams_space) - self.steps_as_tuple = [(name, step.set_hyperparams_space(self.wrapped.get_hyperparams_space())) for name, step in self] - return self + def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Union[Dict, Iterable]: + results = MetaStepMixin.apply_method(self, method=method, step_name=step_name, *kargs, **kwargs) + if len(self) > 0: + for _, step in self: + results.update(step.apply_method(method=method, step_name=step_name, *kargs, **kwargs)) + return results def _will_process(self, data_container: DataContainer, context: ExecutionContext) -> ('BaseStep', DataContainer): data_container, context = BaseStep._will_process(self, data_container, context) @@ -285,6 +266,14 @@ def __iter__(self): """ return iter(self.steps_as_tuple) + def __len__(self): + """ + Get number of steps cloned for each data input. + + :return: len(self.steps_as_tuple) + """ + return len(self.steps_as_tuple) + class FlattenForEach(ForceHandleMixin, ResumableStepMixin, MetaStepMixin, BaseStep): """ diff --git a/testing/steps/test_step_cloner_for_each_data_input.py b/testing/steps/test_step_cloner_for_each_data_input.py index 896db560..7c335d6b 100644 --- a/testing/steps/test_step_cloner_for_each_data_input.py +++ b/testing/steps/test_step_cloner_for_each_data_input.py @@ -159,3 +159,7 @@ def test_step_cloner_should_load_sub_steps(tmpdir): def _create_data(shape): data_inputs = np.random.random(shape).astype(np.float32) return data_inputs + + +def test_step_cloner_should_set_train(tmpdir): + pass From 44e6326110e6f05d1be14dae96c13ec388d4b71d Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 4 Apr 2020 21:12:48 -0400 Subject: [PATCH 02/48] Add Unit Tests For Pipeline Hyperparams --- .../hyperparams/test_get_set_hyperparams.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/testing/hyperparams/test_get_set_hyperparams.py b/testing/hyperparams/test_get_set_hyperparams.py index 5af68b03..b1a3da72 100644 --- a/testing/hyperparams/test_get_set_hyperparams.py +++ b/testing/hyperparams/test_get_set_hyperparams.py @@ -1,6 +1,7 @@ from neuraxle.base import MetaStepMixin, BaseStep, NonFittableMixin, NonTransformableMixin from neuraxle.hyperparams.distributions import RandInt, Boolean from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples +from neuraxle.pipeline import Pipeline from neuraxle.steps.loop import StepClonerForEachDataInput from testing.test_pipeline import SomeStep @@ -181,6 +182,88 @@ def test_step_cloner_should_get_hyperparams_space(): RAND_INT_META_STEP = RandInt(0, 10) +def test_pipeline_should_set_hyperparams(): + p = Pipeline([ + SomeStep().set_name('step_1'), + SomeStep().set_name('step_2') + ]) + + p.set_hyperparams(HyperparameterSamples({ + 'hp': 1, + 'step_1__hp': 2, + 'step_2__hp': 3 + })) + + assert isinstance(p.hyperparams, HyperparameterSpace) + assert p.hyperparams['hp'] == 1 + assert p[0].hyperparams['hp'] == 2 + assert p[1].hyperparams['hp'] == 3 + + +def test_pipeline_should_set_hyperparams_space(): + p = Pipeline([ + SomeStep().set_name('step_1'), + SomeStep().set_name('step_2') + ]) + + p.set_hyperparams_space(HyperparameterSpace({ + 'hp': RandInt(1, 2), + 'step_1__hp': RandInt(2, 3), + 'step_2__hp': RandInt(3, 4) + })) + + assert isinstance(p.hyperparams, HyperparameterSpace) + assert p.hyperparams_space['hp'] == RandInt(1, 2) + assert p[0].hyperparams_space['hp'] == RandInt(2, 3) + assert p[1].hyperparams_space['hp'] == RandInt(3, 4) + + +def test_pipeline_should_update_hyperparams(): + p = Pipeline([ + SomeStep().set_name('step_1'), + SomeStep().set_name('step_2') + ]) + + p.set_hyperparams(HyperparameterSamples({ + 'hp': 1, + 'step_1__hp': 2, + 'step_2__hp': 3 + })) + + p.set_hyperparams(HyperparameterSamples({ + 'hp': 4, + 'step_2__hp': 6 + })) + + assert isinstance(p.hyperparams, HyperparameterSpace) + assert p.hyperparams['hp'] == 4 + assert p[0].hyperparams['hp'] == 2 + assert p[1].hyperparams['hp'] == 6 + + +def test_pipeline_should_update_hyperparams_space(): + p = Pipeline([ + SomeStep().set_name('step_1'), + SomeStep().set_name('step_2') + ]) + + p.set_hyperparams_space(HyperparameterSpace({ + 'hp': RandInt(1, 2), + 'step_1__hp': RandInt(2, 3), + 'step_2__hp': RandInt(3, 4) + })) + p.update_hyperparams_space(HyperparameterSpace({ + 'hp': RandInt(4, 6), + 'step_1__hp': RandInt(2, 3), + 'step_2__hp': RandInt(6, 8) + })) + + assert isinstance(p.hyperparams, HyperparameterSpace) + assert p.hyperparams_space['hp'] == RandInt(4, 6) + assert p[0].hyperparams_space['hp'] == RandInt(2, 3) + assert p[1].hyperparams_space['hp'] == RandInt(6, 8) + + def test_meta_step_mixin_should_get_hyperparams(): p = SomeMetaStepMixin(SomeStep()) p.set_hyperparams(HyperparameterSamples({ From 3ed8ebe7c5fcfab8ab84767988b81337a5feec7a Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 4 Apr 2020 21:13:57 -0400 Subject: [PATCH 03/48] Fix Recursive Get Set Hyperparams Space --- neuraxle/base.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 56f66435..1254dfdd 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -740,14 +740,17 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': self.invalidate() remainders = dict() + step_hyperparams = dict() for name, hparams in hyperparams.items(): if RECURSIVE_STEP_SEPARATOR in name: - if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): - self.hyperparams = HyperparameterSamples(hyperparams[hparams]) + if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): + step_hyperparams[name] = hparams else: remainders[name] = hparams else: - self.hyperparams = HyperparameterSamples(hyperparams[hparams]) + step_hyperparams[name] = hparams + + self.hyperparams = HyperparameterSamples(step_hyperparams) self.apply(method_name='set_hyperparams', hyperparams=remainders, children_only=True) return self @@ -784,15 +787,17 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': self.hyperparams = HyperparameterSamples(self.hyperparams).to_flat() remainders = dict() + step_hyperparams = dict() for name, hparams in hyperparams.items(): if RECURSIVE_STEP_SEPARATOR in name: - if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): - self.hyperparams.update(HyperparameterSamples(hparams).to_flat()) + if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): + step_hyperparams[name] = hparams else: remainders[name] = hparams else: - self.hyperparams.update(HyperparameterSamples(hparams).to_flat()) + step_hyperparams[name] = hparams + self.hyperparams.update(step_hyperparams) self.apply(method_name='update_hyperparams', hyperparams=remainders, children_only=True) return self @@ -867,15 +872,17 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base self.invalidate() remainders = dict() + step_hyperparams_space = dict() for name, hparams_space in hyperparams_space.items(): if RECURSIVE_STEP_SEPARATOR in name: - if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): - self.hyperparams_space = HyperparameterSpace(hparams_space).to_flat() + if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): + step_hyperparams_space[name] = hparams_space else: remainders[name] = hparams_space else: - self.hyperparams_space = HyperparameterSpace(hyperparams_space).to_flat() + step_hyperparams_space[name] = hparams_space + self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) self.apply(method_name='set_hyperparams_space', hyperparams_space=remainders, children_only=True) return self @@ -911,15 +918,17 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B self.hyperparams_space = HyperparameterSamples(self.hyperparams_space).to_flat() remainders = dict() + step_hyperparams_space = dict() for name, hparams_space in hyperparams_space.items(): if RECURSIVE_STEP_SEPARATOR in name: - if name.split(RECURSIVE_STEP_SEPARATOR)[-1].starts_with(self.name): - self.hyperparams_space.update(HyperparameterSpace(hparams_space).to_flat()) + if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): + step_hyperparams_space[name] = hparams_space else: remainders[name] = hparams_space else: - self.hyperparams_space.update(HyperparameterSpace(hparams_space).to_flat()) + step_hyperparams_space[name] = hparams_space + self.hyperparams_space.update(HyperparameterSpace(step_hyperparams_space).to_flat()) self.apply(method_name='update_hyperparams_space', hyperparams_space=remainders, children_only=True) return self @@ -982,7 +991,7 @@ def apply_method(self, method: Callable, step_name=None, children_only: bool=Fal return {} if step_name is not None: - step_name = "{}__{}".format(step_name, self.name) + step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) else: step_name = self.name @@ -1006,7 +1015,7 @@ def apply(self, method_name: str, step_name=None, children_only: bool=False, *ka return results if step_name is not None: - step_name = "{}__{}".format(step_name, self.name) + step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) else: step_name = self.name @@ -2347,7 +2356,7 @@ def apply(self, method_name: str, step_name=None, children_only: bool = False, * results = BaseStep.apply(self, method_name, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: - step_name = "{}__{}".format(step_name, self.name) + step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) else: step_name = self.name @@ -2371,7 +2380,7 @@ def apply_method(self, method: Callable, step_name=None, children_only=False, *k results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: - step_name = "{}__{}".format(step_name, self.name) + step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) else: step_name = self.name From cab284a6063036527652563169ad87268524b501 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 4 Apr 2020 22:23:11 -0400 Subject: [PATCH 04/48] Fix Base Step Recursive Get/Set Hyperparams & Remove Old Get/Set Hyperparams In Other Steps --- neuraxle/base.py | 142 +++--------------- .../hyperparams/test_get_set_hyperparams.py | 41 +++-- 2 files changed, 46 insertions(+), 137 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 1254dfdd..4217a309 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -738,13 +738,15 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ self.invalidate() + hyperparams = HyperparameterSamples(hyperparams).to_flat() remainders = dict() step_hyperparams = dict() for name, hparams in hyperparams.items(): if RECURSIVE_STEP_SEPARATOR in name: - if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): - step_hyperparams[name] = hparams + name_split = name.split(RECURSIVE_STEP_SEPARATOR) + if str(name_split[-2]).startswith(self.name): + step_hyperparams[name_split[-1]] = hparams else: remainders[name] = hparams else: @@ -783,15 +785,15 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ - self.hyperparams.update(hyperparams) - self.hyperparams = HyperparameterSamples(self.hyperparams).to_flat() + hyperparams = HyperparameterSamples(hyperparams).to_flat() remainders = dict() step_hyperparams = dict() for name, hparams in hyperparams.items(): if RECURSIVE_STEP_SEPARATOR in name: - if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): - step_hyperparams[name] = hparams + name_split = name.split(RECURSIVE_STEP_SEPARATOR) + if str(name_split[-2]).startswith(self.name): + step_hyperparams[name_split[-1]] = hparams else: remainders[name] = hparams else: @@ -870,13 +872,15 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ self.invalidate() - + hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() remainders = dict() step_hyperparams_space = dict() + for name, hparams_space in hyperparams_space.items(): if RECURSIVE_STEP_SEPARATOR in name: - if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): - step_hyperparams_space[name] = hparams_space + name_split = name.split(RECURSIVE_STEP_SEPARATOR) + if str(name_split[-2]).startswith(self.name): + step_hyperparams_space[name_split[-1]] = hparams_space else: remainders[name] = hparams_space else: @@ -914,15 +918,15 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - self.hyperparams_space.update(hyperparams_space) - self.hyperparams_space = HyperparameterSamples(self.hyperparams_space).to_flat() - + hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() remainders = dict() step_hyperparams_space = dict() + for name, hparams_space in hyperparams_space.items(): if RECURSIVE_STEP_SEPARATOR in name: - if str(name.split(RECURSIVE_STEP_SEPARATOR)[-1]).startswith(self.name): - step_hyperparams_space[name] = hparams_space + name_split = name.split(RECURSIVE_STEP_SEPARATOR) + if str(name_split[-2]).startswith(self.name): + step_hyperparams_space[name_split[-1]] = hparams_space else: remainders[name] = hparams_space else: @@ -1723,67 +1727,6 @@ def teardown(self) -> BaseStep: self.is_initialized = False return self - def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: - """ - Set step hyperparameters, and wrapped step hyperparams with the given hyperparams. - - Example : - - .. code-block:: python - - step.set_hyperparams(HyperparameterSamples({ - 'learning_rate': 0.10 - 'wrapped__learning_rate': 0.10 # this will set the wrapped step 'learning_rate' hyperparam - })) - - :param hyperparams: hyperparameters - :return: self - - .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - self.invalidate() - - hyperparams: HyperparameterSamples = HyperparameterSamples(hyperparams).to_nested_dict() - - remainders = dict() - for name, hparams in hyperparams.items(): - if name == self.wrapped.name: - self.wrapped.set_hyperparams(hparams) - else: - remainders[name] = hparams - - self.hyperparams = HyperparameterSamples(remainders) - - return self - - def update_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: - """ - Update the step, and the wrapped step hyperparams without removing the already set hyperparameters. - Please refer to :func:`~BaseStep.update_hyperparams`. - - :param hyperparams: hyperparameters - :return: self - - .. seealso:: - :func:`~BaseStep.update_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - self.invalidate() - - hyperparams: HyperparameterSamples = HyperparameterSamples(hyperparams).to_nested_dict() - - remainders = dict() - for name, hparams in hyperparams.items(): - if name == self.wrapped.name: - self.wrapped.update_hyperparams(hparams) - else: - remainders[name] = hparams - - self.hyperparams.update(remainders) - - return self - def get_hyperparams(self) -> HyperparameterSamples: """ Get step hyperparameters as :class:`~neuraxle.hyperparams.space.HyperparameterSamples` with flattened hyperparams. @@ -1798,55 +1741,6 @@ def get_hyperparams(self) -> HyperparameterSamples: self.wrapped.name: self.wrapped.get_hyperparams().to_flat_as_dict_primitive() }).to_flat() - def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': - """ - Set meta step and wrapped step hyperparams space using the given hyperparams space. - - :param hyperparams_space: ordered dict containing all hyperparameter spaces - :return: self - """ - self.invalidate() - - hyperparams_space: HyperparameterSpace = HyperparameterSpace(hyperparams_space).to_nested_dict() - - remainders = dict() - for name, hparams in hyperparams_space.items(): - if name == self.wrapped.name: - self.wrapped.set_hyperparams_space(hparams) - else: - remainders[name] = hparams - - self.hyperparams_space = HyperparameterSpace(remainders) - - return self - - def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> BaseStep: - """ - Update the step, and the wrapped step hyperparams without removing the already set hyperparameters. - Please refer to :func:`~BaseStep.update_hyperparams`. - - :param hyperparams_space: hyperparameters - :return: self - - .. seealso:: - :func:`~BaseStep.update_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - self.is_invalidated = True - - hyperparams_space: HyperparameterSpace = HyperparameterSpace(hyperparams_space).to_nested_dict() - - remainders = dict() - for name, hparams_space in hyperparams_space.items(): - if name == self.wrapped.name: - self.wrapped.update_hyperparams_space(hparams_space) - else: - remainders[name] = hparams_space - - self.hyperparams_space.update(remainders) - - return self - def get_hyperparams_space(self) -> HyperparameterSpace: """ Get meta step and wrapped step hyperparams as a flat hyperparameter space diff --git a/testing/hyperparams/test_get_set_hyperparams.py b/testing/hyperparams/test_get_set_hyperparams.py index b1a3da72..cd824537 100644 --- a/testing/hyperparams/test_get_set_hyperparams.py +++ b/testing/hyperparams/test_get_set_hyperparams.py @@ -24,7 +24,11 @@ class SomeMetaStepMixin(NonTransformableMixin, NonFittableMixin, MetaStepMixin, BaseStep): - pass + def __init__(self, wrapped: BaseStep): + BaseStep.__init__(self) + NonTransformableMixin.__init__(self) + NonFittableMixin.__init__(self) + MetaStepMixin.__init__(self, wrapped) class SomeStepInverseTransform(SomeStep): @@ -194,7 +198,7 @@ def test_pipeline_should_set_hyperparams(): 'step_2__hp': 3 })) - assert isinstance(p.hyperparams, HyperparameterSpace) + assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams['hp'] == 1 assert p[0].hyperparams['hp'] == 2 assert p[1].hyperparams['hp'] == 3 @@ -212,10 +216,16 @@ def test_pipeline_should_set_hyperparams_space(): 'step_2__hp': RandInt(3, 4) })) - assert isinstance(p.hyperparams, HyperparameterSpace) - assert p.hyperparams_space['hp'] == RandInt(1, 2) - assert p[0].hyperparams_space['hp'] == RandInt(2, 3) - assert p[1].hyperparams_space['hp'] == RandInt(3, 4) + assert isinstance(p.hyperparams_space, HyperparameterSpace) + + assert p.hyperparams_space['hp'].min_included == 1 + assert p.hyperparams_space['hp'].max_included == 2 + + assert p[0].hyperparams_space['hp'].min_included == 2 + assert p[0].hyperparams_space['hp'].max_included == 3 + + assert p[1].hyperparams_space['hp'].min_included == 3 + assert p[1].hyperparams_space['hp'].max_included == 4 def test_pipeline_should_update_hyperparams(): @@ -230,12 +240,12 @@ def test_pipeline_should_update_hyperparams(): 'step_2__hp': 3 })) - p.set_hyperparams(HyperparameterSamples({ + p.update_hyperparams(HyperparameterSamples({ 'hp': 4, 'step_2__hp': 6 })) - assert isinstance(p.hyperparams, HyperparameterSpace) + assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams['hp'] == 4 assert p[0].hyperparams['hp'] == 2 assert p[1].hyperparams['hp'] == 6 @@ -254,14 +264,19 @@ def test_pipeline_should_update_hyperparams_space(): })) p.update_hyperparams_space(HyperparameterSpace({ 'hp': RandInt(4, 6), - 'step_1__hp': RandInt(2, 3), 'step_2__hp': RandInt(6, 8) })) - assert isinstance(p.hyperparams, HyperparameterSpace) - assert p.hyperparams_space['hp'] == RandInt(4, 6) - assert p[0].hyperparams_space['hp'] == RandInt(2, 3) - assert p[1].hyperparams_space['hp'] == RandInt(6, 8) + assert isinstance(p.hyperparams_space, HyperparameterSpace) + + assert p.hyperparams_space['hp'].min_included == 4 + assert p.hyperparams_space['hp'].max_included == 6 + + assert p[0].hyperparams_space['hp'].min_included == 2 + assert p[0].hyperparams_space['hp'].max_included == 3 + + assert p[1].hyperparams_space['hp'].min_included == 6 + assert p[1].hyperparams_space['hp'].max_included == 8 def test_meta_step_mixin_should_get_hyperparams(): From 8d42cfd06db3bf4bf25ef7025506a98ed40f5baf Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 4 Apr 2020 22:53:11 -0400 Subject: [PATCH 05/48] Fix SKLearnWrapper set_hyperparams --- neuraxle/steps/sklearn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index 5b331afb..22467617 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -78,8 +78,9 @@ def transform(self, data_inputs): return self.wrapped_sklearn_predictor.predict(data_inputs) return self.wrapped_sklearn_predictor.transform(data_inputs) - def set_hyperparams(self, flat_hyperparams: HyperparameterSamples) -> BaseStep: - BaseStep.set_hyperparams(self, flat_hyperparams) + def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: + BaseStep.set_hyperparams(self, hyperparams) + flat_hyperparams = HyperparameterSamples(self.hyperparams).to_flat() self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(flat_hyperparams).to_flat_as_dict_primitive()) return self From f40063a713be32b9ea6b1d7ae6dc3d4e507e477d Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 4 Apr 2020 23:08:31 -0400 Subject: [PATCH 06/48] Don't set hyperparams of sub steps in BaseStep recursive set hyperparams & hyperparams_space methods --- neuraxle/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 4217a309..10662605 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -752,7 +752,7 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': else: step_hyperparams[name] = hparams - self.hyperparams = HyperparameterSamples(step_hyperparams) + self.hyperparams = HyperparameterSamples(step_hyperparams) if len(step_hyperparams) > 0 else self.hyperparams self.apply(method_name='set_hyperparams', hyperparams=remainders, children_only=True) return self @@ -886,8 +886,9 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base else: step_hyperparams_space[name] = hparams_space - self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) + self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) if len(step_hyperparams_space) > 0 else self.hyperparams_space self.apply(method_name='set_hyperparams_space', hyperparams_space=remainders, children_only=True) + return self def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': @@ -934,6 +935,7 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B self.hyperparams_space.update(HyperparameterSpace(step_hyperparams_space).to_flat()) self.apply(method_name='update_hyperparams_space', hyperparams_space=remainders, children_only=True) + return self def get_hyperparams_space(self) -> HyperparameterSpace: From bbfcaaf5641cf0abe2757cc1e5dc94e1d97e5cbc Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 4 Apr 2020 23:46:31 -0400 Subject: [PATCH 07/48] Extract Method For Recursive Step Hyperparams Values, And Remainders --- neuraxle/base.py | 68 ++++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 10662605..5a4bd214 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -738,23 +738,14 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ self.invalidate() - hyperparams = HyperparameterSamples(hyperparams).to_flat() - remainders = dict() - step_hyperparams = dict() - for name, hparams in hyperparams.items(): - if RECURSIVE_STEP_SEPARATOR in name: - name_split = name.split(RECURSIVE_STEP_SEPARATOR) - if str(name_split[-2]).startswith(self.name): - step_hyperparams[name_split[-1]] = hparams - else: - remainders[name] = hparams - else: - step_hyperparams[name] = hparams + hyperparams = HyperparameterSamples(hyperparams).to_flat() + step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams) self.hyperparams = HyperparameterSamples(step_hyperparams) if len(step_hyperparams) > 0 else self.hyperparams self.apply(method_name='set_hyperparams', hyperparams=remainders, children_only=True) + return self def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': @@ -787,21 +778,32 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': """ hyperparams = HyperparameterSamples(hyperparams).to_flat() + step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams) + self.hyperparams.update(step_hyperparams) + + self.apply(method_name='update_hyperparams', hyperparams=remainders, children_only=True) + + return self + + def _create_recursive_step_values_and_remainders(self, hyperparams): remainders = dict() - step_hyperparams = dict() + step_values_dict = dict() + for name, hparams in hyperparams.items(): if RECURSIVE_STEP_SEPARATOR in name: name_split = name.split(RECURSIVE_STEP_SEPARATOR) - if str(name_split[-2]).startswith(self.name): - step_hyperparams[name_split[-1]] = hparams + + if str(name_split[-2]) == self.name and len(name_split) == 2: + step_values_dict[name_split[-1]] = hparams + elif name_split[0] == self.name: + remainders[RECURSIVE_STEP_SEPARATOR.join(name_split[1:])] = hparams else: remainders[name] = hparams + else: - step_hyperparams[name] = hparams + step_values_dict[name] = hparams - self.hyperparams.update(step_hyperparams) - self.apply(method_name='update_hyperparams', hyperparams=remainders, children_only=True) - return self + return step_values_dict, remainders def get_hyperparams(self) -> HyperparameterSamples: """ @@ -873,20 +875,10 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base """ self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() - remainders = dict() - step_hyperparams_space = dict() - - for name, hparams_space in hyperparams_space.items(): - if RECURSIVE_STEP_SEPARATOR in name: - name_split = name.split(RECURSIVE_STEP_SEPARATOR) - if str(name_split[-2]).startswith(self.name): - step_hyperparams_space[name_split[-1]] = hparams_space - else: - remainders[name] = hparams_space - else: - step_hyperparams_space[name] = hparams_space + step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space) self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) if len(step_hyperparams_space) > 0 else self.hyperparams_space + self.apply(method_name='set_hyperparams_space', hyperparams_space=remainders, children_only=True) return self @@ -920,20 +912,10 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() - remainders = dict() - step_hyperparams_space = dict() - - for name, hparams_space in hyperparams_space.items(): - if RECURSIVE_STEP_SEPARATOR in name: - name_split = name.split(RECURSIVE_STEP_SEPARATOR) - if str(name_split[-2]).startswith(self.name): - step_hyperparams_space[name_split[-1]] = hparams_space - else: - remainders[name] = hparams_space - else: - step_hyperparams_space[name] = hparams_space + step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space) self.hyperparams_space.update(HyperparameterSpace(step_hyperparams_space).to_flat()) + self.apply(method_name='update_hyperparams_space', hyperparams_space=remainders, children_only=True) return self From 643331bc9d4c4ecc2d83fa5921ed83de37833a5a Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 5 Apr 2020 09:56:02 -0400 Subject: [PATCH 08/48] Use Apply For Recursive Getter Methods --- neuraxle/base.py | 125 ++++-------------- .../hyperparams/test_get_set_hyperparams.py | 36 +++-- 2 files changed, 51 insertions(+), 110 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 5a4bd214..ba61f709 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -814,7 +814,16 @@ def get_hyperparams(self) -> HyperparameterSamples: .. seealso:: * :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ - return self.hyperparams + results = self.apply(method_name='get_hyperparams', children_only=True) + results = HyperparameterSamples({ + RECURSIVE_STEP_SEPARATOR.join(key.split(RECURSIVE_STEP_SEPARATOR)[1:]): value.to_flat() + for key, value in results.items() + }).to_flat() + + return HyperparameterSamples({ + **self.hyperparams.to_flat_as_dict_primitive(), + **results + }) def set_params(self, **params) -> 'BaseStep': """ @@ -930,13 +939,23 @@ def get_hyperparams_space(self) -> HyperparameterSpace: step.get_hyperparams_space() + :return: step hyperparams space .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ - return self.hyperparams_space + results = self.apply(method_name='get_hyperparams_space', children_only=True) + results = HyperparameterSpace({ + RECURSIVE_STEP_SEPARATOR.join(key.split(RECURSIVE_STEP_SEPARATOR)[1:]): value.to_flat() + for key, value in results.items() + }).to_flat() + + return HyperparameterSpace({ + **self.hyperparams_space, + **results + }) def handle_inverse_transform(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: """ @@ -1711,31 +1730,6 @@ def teardown(self) -> BaseStep: self.is_initialized = False return self - def get_hyperparams(self) -> HyperparameterSamples: - """ - Get step hyperparameters as :class:`~neuraxle.hyperparams.space.HyperparameterSamples` with flattened hyperparams. - - :return: step hyperparameters - - .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - return HyperparameterSamples({ - **self.hyperparams.to_flat_as_dict_primitive(), - self.wrapped.name: self.wrapped.get_hyperparams().to_flat_as_dict_primitive() - }).to_flat() - - def get_hyperparams_space(self) -> HyperparameterSpace: - """ - Get meta step and wrapped step hyperparams as a flat hyperparameter space - - :return: hyperparameters_space - """ - return HyperparameterSpace({ - **self.hyperparams_space.to_flat_as_dict_primitive(), - self.wrapped.name: self.wrapped.get_hyperparams_space().to_flat_as_dict_primitive() - }).to_flat() - def get_step(self) -> BaseStep: """ Get wrapped step @@ -1840,7 +1834,7 @@ def apply(self, method_name: str, step_name=None, children_only=False, *kargs, * results = BaseStep.apply(self, method_name=method_name, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: - step_name = "{}__{}".format(step_name, self.name) + step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) else: step_name = self.name @@ -1864,7 +1858,7 @@ def apply_method(self, method: Callable, step_name=None, children_only=False, *k results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, **kwargs) if step_name is not None: - step_name = "{}__{}".format(step_name, self.name) + step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) else: step_name = self.name @@ -2358,79 +2352,6 @@ def _refresh_steps(self): for name, step in self.items(): step.name = name - def get_hyperparams(self) -> HyperparameterSamples: - """ - Get step hyperparameters as :class:`~neuraxle.space.HyperparameterSamples`. - - Example : - - .. code-block:: python - - p = Pipeline([SomeStep()]) - p.set_hyperparams(HyperparameterSamples({ - 'learning_rate': 0.1, - 'some_step__learning_rate': 0.2 # will set SomeStep() hyperparam 'learning_rate' to 0.2 - })) - - hp = p.get_hyperparams() - # hp ==> { 'learning_rate': 0.1, 'some_step__learning_rate': 0.2 } - - :return: step hyperparameters - - .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - hyperparams = dict() - - for k, v in self.steps.items(): - hparams = v.get_hyperparams() # TODO: oop diamond problem? - if hasattr(v, "hyperparams"): - hparams.update(v.hyperparams) - if len(hparams) > 0: - hyperparams[k] = hparams - - hyperparams = HyperparameterSamples(hyperparams) - - hyperparams.update( - BaseStep.get_hyperparams(self) - ) - - return hyperparams.to_flat() - - def get_hyperparams_space(self): - """ - Get step hyperparameters space as :class:`~neuraxle.space.HyperparameterSpace`. - - Example : - - .. code-block:: python - - p = Pipeline([SomeStep()]) - p.set_hyperparams_space(HyperparameterSpace({ - 'learning_rate': RandInt(0,5), - 'some_step__learning_rate': RandInt(0, 10) # will set SomeStep() 'learning_rate' hyperparam space to RandInt(0, 10) - })) - - hp = p.get_hyperparams_space() - # hp ==> { 'learning_rate': RandInt(0,5), 'some_step__learning_rate': RandInt(0,10) } - - :return: step hyperparameters space - - .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSpace` - """ - all_hyperparams = HyperparameterSpace() - for step_name, step in self.steps_as_tuple: - hspace = step.get_hyperparams_space() - all_hyperparams.update({ - step_name: hspace - }) - all_hyperparams.update( - BaseStep.get_hyperparams_space(self) - ) - - return all_hyperparams.to_flat() - def should_save(self): """ Returns if the step needs to be saved or not. diff --git a/testing/hyperparams/test_get_set_hyperparams.py b/testing/hyperparams/test_get_set_hyperparams.py index cd824537..36edb8f9 100644 --- a/testing/hyperparams/test_get_set_hyperparams.py +++ b/testing/hyperparams/test_get_set_hyperparams.py @@ -204,28 +204,48 @@ def test_pipeline_should_set_hyperparams(): assert p[1].hyperparams['hp'] == 3 -def test_pipeline_should_set_hyperparams_space(): +def test_pipeline_should_get_hyperparams(): p = Pipeline([ SomeStep().set_name('step_1'), SomeStep().set_name('step_2') ]) + p.set_hyperparams(HyperparameterSamples({ + 'hp': 1, + 'step_1__hp': 2, + 'step_2__hp': 3 + })) + + hyperparams = p.get_hyperparams() + + assert isinstance(hyperparams, HyperparameterSamples) + assert hyperparams['hp'] == 1 + assert hyperparams['step_1__hp'] == 2 + assert hyperparams['step_2__hp'] == 3 + +def test_pipeline_should_get_hyperparams_space(): + p = Pipeline([ + SomeStep().set_name('step_1'), + SomeStep().set_name('step_2') + ]) p.set_hyperparams_space(HyperparameterSpace({ 'hp': RandInt(1, 2), 'step_1__hp': RandInt(2, 3), 'step_2__hp': RandInt(3, 4) })) - assert isinstance(p.hyperparams_space, HyperparameterSpace) + hyperparams_space = p.get_hyperparams_space() - assert p.hyperparams_space['hp'].min_included == 1 - assert p.hyperparams_space['hp'].max_included == 2 + assert isinstance(hyperparams_space, HyperparameterSpace) - assert p[0].hyperparams_space['hp'].min_included == 2 - assert p[0].hyperparams_space['hp'].max_included == 3 + assert hyperparams_space['hp'].min_included == 1 + assert hyperparams_space['hp'].max_included == 2 + + assert hyperparams_space['step_1__hp'].min_included == 2 + assert hyperparams_space['step_1__hp'].max_included == 3 - assert p[1].hyperparams_space['hp'].min_included == 3 - assert p[1].hyperparams_space['hp'].max_included == 4 + assert hyperparams_space['step_2__hp'].min_included == 3 + assert hyperparams_space['step_2__hp'].max_included == 4 def test_pipeline_should_update_hyperparams(): From 309b13584cd252b930957799f556d827c9de5c8f Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 5 Apr 2020 10:57:18 -0400 Subject: [PATCH 09/48] Somewhat Fix Pipeline To SKLearn Test With Set Params Recursive Method --- neuraxle/steps/sklearn.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index 22467617..4b57e2a6 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -80,7 +80,13 @@ def transform(self, data_inputs): def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: BaseStep.set_hyperparams(self, hyperparams) - flat_hyperparams = HyperparameterSamples(self.hyperparams).to_flat() + + # this should work + flat_hyperparams = { + '__'.join(key.split('__')[1:]): value for key, value in HyperparameterSamples(hyperparams).to_flat().items() + } + + flat_hyperparams = HyperparameterSamples(flat_hyperparams).to_flat() self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(flat_hyperparams).to_flat_as_dict_primitive()) return self From 6dfe54825252a591fcd08a4de225b68493e6852f Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 6 May 2020 12:50:51 -0400 Subject: [PATCH 10/48] Set Root Hyperparams, And HyperparameterSapces Properly In Recursive Apply Logic --- neuraxle/base.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index ba61f709..92ad6f3b 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -737,14 +737,18 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ + self._set_hyperparams(hyperparams=hyperparams, root=True) + return self + + def _set_hyperparams(self, hyperparams: HyperparameterSamples, root=False) -> 'BaseStep': self.invalidate() hyperparams = HyperparameterSamples(hyperparams).to_flat() - step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams) + step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams, root) self.hyperparams = HyperparameterSamples(step_hyperparams) if len(step_hyperparams) > 0 else self.hyperparams - self.apply(method_name='set_hyperparams', hyperparams=remainders, children_only=True) + self.apply(method_name='_set_hyperparams', hyperparams=remainders, children_only=True) return self @@ -776,16 +780,20 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ + self._update_hyperparams(hyperparams=hyperparams, root=True) + return self + + def _update_hyperparams(self, hyperparams: HyperparameterSamples, root=False) -> 'BaseStep': hyperparams = HyperparameterSamples(hyperparams).to_flat() - step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams) + step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams, root) self.hyperparams.update(step_hyperparams) - self.apply(method_name='update_hyperparams', hyperparams=remainders, children_only=True) + self.apply(method_name='_update_hyperparams', hyperparams=remainders, children_only=True) return self - def _create_recursive_step_values_and_remainders(self, hyperparams): + def _create_recursive_step_values_and_remainders(self, hyperparams, root=False): remainders = dict() step_values_dict = dict() @@ -797,7 +805,7 @@ def _create_recursive_step_values_and_remainders(self, hyperparams): step_values_dict[name_split[-1]] = hparams elif name_split[0] == self.name: remainders[RECURSIVE_STEP_SEPARATOR.join(name_split[1:])] = hparams - else: + elif root: remainders[name] = hparams else: @@ -882,13 +890,17 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ + self._set_hyperparams_space(hyperparams_space=hyperparams_space, root=True) + return self + + def _set_hyperparams_space(self, hyperparams_space: HyperparameterSpace, root=False) -> 'BaseStep': self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() - step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space) + step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space, root) self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) if len(step_hyperparams_space) > 0 else self.hyperparams_space - self.apply(method_name='set_hyperparams_space', hyperparams_space=remainders, children_only=True) + self.apply(method_name='_set_hyperparams_space', hyperparams_space=remainders, children_only=True) return self @@ -920,12 +932,16 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ + self._update_hyperparams_space(hyperparams_space=hyperparams_space, root=True) + return self + + def _update_hyperparams_space(self, hyperparams_space: HyperparameterSpace, root=False) -> 'BaseStep': hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() - step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space) + step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space, root) self.hyperparams_space.update(HyperparameterSpace(step_hyperparams_space).to_flat()) - self.apply(method_name='update_hyperparams_space', hyperparams_space=remainders, children_only=True) + self.apply(method_name='_update_hyperparams_space', hyperparams_space=remainders, children_only=True) return self From 8153aeac78f9b492a04ca7274e1cc8d1abb58a8d Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 6 May 2020 13:22:39 -0400 Subject: [PATCH 11/48] Fix sklearn set hyperparams(params) with recursive methods --- neuraxle/steps/sklearn.py | 15 +++++---------- testing/test_pipeline.py | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index 4b57e2a6..974c0c8d 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -78,16 +78,11 @@ def transform(self, data_inputs): return self.wrapped_sklearn_predictor.predict(data_inputs) return self.wrapped_sklearn_predictor.transform(data_inputs) - def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: - BaseStep.set_hyperparams(self, hyperparams) - - # this should work - flat_hyperparams = { - '__'.join(key.split('__')[1:]): value for key, value in HyperparameterSamples(hyperparams).to_flat().items() - } - - flat_hyperparams = HyperparameterSamples(flat_hyperparams).to_flat() - self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(flat_hyperparams).to_flat_as_dict_primitive()) + def _set_hyperparams(self, hyperparams: HyperparameterSamples, root=False) -> BaseStep: + BaseStep._set_hyperparams(self, hyperparams) + step_hyperparams, remainders = BaseStep._create_recursive_step_values_and_remainders(self, hyperparams, root) + self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(step_hyperparams).to_flat_as_dict_primitive()) + self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(HyperparameterSamples(remainders).to_flat()).to_flat_as_dict_primitive()) return self def get_hyperparams(self): diff --git a/testing/test_pipeline.py b/testing/test_pipeline.py index d6539904..d31a49dc 100644 --- a/testing/test_pipeline.py +++ b/testing/test_pipeline.py @@ -297,7 +297,7 @@ def test_pipeline_tosklearn(): "b__learning_rate": 9 } }) - assert the_step.get_hyperparams()["learning_rate"] == 7 + assert p.get_hyperparams()['b__a__z__learning_rate'] == 7 p = p.tosklearn() p = sklearn.pipeline.Pipeline([('sk', p)]) From e653cc0da80533fa992bd8851dc0beb36a8da7f1 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 6 May 2020 18:19:26 -0400 Subject: [PATCH 12/48] Wip Recursive Dict Refactor --- neuraxle/base.py | 98 ++++++++++++++++++-- neuraxle/hyperparams/space.py | 164 +++++++++++++++++++++------------- neuraxle/steps/loop.py | 22 +++++ neuraxle/steps/sklearn.py | 15 +++- 4 files changed, 227 insertions(+), 72 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 92ad6f3b..638377df 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -39,7 +39,7 @@ from sklearn.base import BaseEstimator from neuraxle.data_container import DataContainer -from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples +from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples, RecursiveDict RECURSIVE_STEP_SEPARATOR = '__' @@ -898,7 +898,8 @@ def _set_hyperparams_space(self, hyperparams_space: HyperparameterSpace, root=Fa hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space, root) - self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) if len(step_hyperparams_space) > 0 else self.hyperparams_space + self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) if len( + step_hyperparams_space) > 0 else self.hyperparams_space self.apply(method_name='_set_hyperparams_space', hyperparams_space=remainders, children_only=True) @@ -999,7 +1000,7 @@ def _inverse_transform_data_container( return data_container - def apply_method(self, method: Callable, step_name=None, children_only: bool=False, *kargs, **kwargs) -> Dict: + def apply_method(self, method: Callable, step_name=None, children_only: bool = False, *kargs, **kwargs) -> Dict: """ Apply a method to a step and its children. @@ -1022,7 +1023,7 @@ def apply_method(self, method: Callable, step_name=None, children_only: bool=Fal step_name: method(self, *kargs, **kwargs) } - def apply(self, method_name: str, step_name=None, children_only: bool=False, *kargs, **kwargs) -> Dict: + def apply(self, method_name: str, step_name=None, children_only: bool = False, *kargs, **kwargs) -> Dict: """ Apply a method to a step and its children. @@ -1847,7 +1848,8 @@ def apply(self, method_name: str, step_name=None, children_only=False, *kargs, * :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = BaseStep.apply(self, method_name=method_name, step_name=step_name, children_only=children_only, *kargs, **kwargs) + results = BaseStep.apply(self, method_name=method_name, step_name=step_name, children_only=children_only, + *kargs, **kwargs) if step_name is not None: step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) @@ -1860,7 +1862,8 @@ def apply(self, method_name: str, step_name=None, children_only=False, *kargs, * return results - def apply_method(self, method: Callable, step_name=None, children_only=False, *kargs, **kwargs) -> Union[Dict, Iterable]: + def apply_method(self, method: Callable, step_name=None, children_only=False, *kargs, **kwargs) -> Union[ + Dict, Iterable]: """ Apply method to the meta step and its wrapped step. @@ -1871,7 +1874,8 @@ def apply_method(self, method: Callable, step_name=None, children_only=False, *k :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, **kwargs) + results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, + **kwargs) if step_name is not None: step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) @@ -2265,7 +2269,8 @@ def apply_method(self, method: Callable, step_name=None, children_only=False, *k :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, **kwargs) + results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, + **kwargs) if step_name is not None: step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) @@ -2895,7 +2900,8 @@ def fit_transform(self, data_inputs, expected_outputs=None) -> Tuple['HandleOnly return new_self, data_container.data_inputs - def _encapsulate_data(self, data_inputs, expected_outputs=None, execution_mode=None) -> Tuple[ExecutionContext, DataContainer]: + def _encapsulate_data(self, data_inputs, expected_outputs=None, execution_mode=None) -> Tuple[ + ExecutionContext, DataContainer]: """ Encapsulate data with :class:`~neuraxle.data_container.DataContainer`. @@ -2965,6 +2971,7 @@ class FullDumpLoader(Identity): :class:`BaseStep`, :class:`Identity` """ + def __init__(self, name, stripped_saver=None): if stripped_saver is None: stripped_saver = JoblibStepSaver() @@ -2985,3 +2992,76 @@ def load(self, context: ExecutionContext, full_dump=True) -> BaseStep: context.pop() return loaded_self.load(context, full_dump) + + +class StepsWithChildrenMixin: + def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + """ + Apply method to root, and children steps. + Split the root, and children values inside the arguments of type RecursiveDict. + + :param method_name: + :param step_name: + :param kargs: + :param kwargs: + :return: + """ + root_kwargs, children_kwargs = self._get_root_and_children_kwargs(**kwargs) + root_kargs, children_kargs = self._get_root_and_children_kargs(*kargs) + + results = BaseStep.apply(self, method_name=method_name, step_name=step_name, *root_kargs, **root_kwargs) + children_results = self.apply_to_children(self, method_name=method_name, step_name=step_name, *root_kargs, **root_kwargs) + results.update(children_results) + + return results + + def _get_root_and_children_kargs(self, *kargs): + """ + Split root, and children values inside *kargs. + + :param kargs: kargs values + :return: + """ + root_kargs = list() + children_kargs = list() + for arg in kargs: + if isinstance(arg, RecursiveDict): + root_kargs.append(arg.get_root_values(self.name)) + children_kargs.append(arg.get_children_values(self.name)) + else: + root_kargs.append(arg) + children_kargs.append(arg) + + return root_kargs, children_kargs + + def _get_root_and_children_kwargs(self, **kwargs): + """ + Split root, and children values inside *kwargs. + + :param kwargs: kwargs values + :return: + """ + root_kwargs = dict() + children_kwargs = dict() + for key in kwargs: + if isinstance(kwargs[key], RecursiveDict): + root_kwargs[key] = kwargs[key].get_root_values(self.name) + children_kwargs[key] = kwargs[key].get_children_values(self.name) + else: + root_kwargs[key] = kwargs[key] + children_kwargs[key] = kwargs[key] + + return root_kwargs, children_kwargs + + @abstractmethod + def apply_to_children(self, method_name: str, step_name=None, *kargs, **kwargs): + """ + Base method to apply a method to all sub steps. + + :param method_name: method name that need to be called on all steps + :param step_name: current pipeline step name + :param kargs: any additional arguments to be passed to the method + :param kwargs: any additional positional arguments to be passed to the method + :return: accumulated results + """ + pass diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index eb45aafe..9ff92741 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -69,70 +69,67 @@ PARAMS_SPLIT_SEQ = "__" -def nested_dict_to_flat(nested_hyperparams, dict_ctor=OrderedDict): +class RecursiveDict(OrderedDict): """ - Convert a nested hyperparameter dictionary to a flat one. + Wraps an hyperparameter nested dict or flat dict, and offer a few more functions. - :param nested_hyperparams: a nested hyperparameter dictionary. - :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. - :return: a flat hyperparameter dictionary. + This can be set on a Pipeline with the method ``set_hyperparams``. + + HyperparameterSamples are often the result of calling ``.rvs()`` on an HyperparameterSpace. """ - ret = dict_ctor() - for k, v in nested_hyperparams.items(): - if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor): - _ret = nested_dict_to_flat(v) - for key, val in _ret.items(): - ret[k + PARAMS_SPLIT_SEQ + key] = val - else: - ret[k] = v - return ret + DEFAULT_SEPARATOR = '__' + def __init__(self, separator=None, recursive_dict_constructor=None, *args, **kwds): + super().__init__(*args, **kwds) + if separator is None: + separator = self.DEFAULT_SEPARATOR + if recursive_dict_constructor is None: + separator = RecursiveDict -def flat_to_nested_dict(flat_hyperparams, dict_ctor=OrderedDict): - """ - Convert a flat hyperparameter dictionary to a nested one. + self.separator = separator + self.recursive_dict_constructor = recursive_dict_constructor - :param flat_hyperparams: a flat hyperparameter dictionary. - :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. - :return: a nested hyperparameter dictionary. - """ - pre_ret = dict_ctor() - ret = dict_ctor() - for k, v in flat_hyperparams.items(): - k, _, key = k.partition(PARAMS_SPLIT_SEQ) - if len(key) > 0: - if k not in pre_ret.keys(): - pre_ret[k] = dict_ctor() - pre_ret[k][key] = v - else: - ret[k] = v - for k, v in pre_ret.items(): - ret[k] = flat_to_nested_dict(v) - return ret + def get_root_values(self, root_name: str): + root_dict_values = dict() + for name, hparams in self.items(): + if self.separator in name: + name_split = name.split(self.separator) + if str(name_split[-2]) == root_name and len(name_split) == 2: + root_dict_values[name_split[-1]] = hparams + else: + root_dict_values[name] = hparams -class HyperparameterSamples(OrderedDict): - """Wraps an hyperparameter nested dict or flat dict, and offer a few more functions. + return root_dict_values - This can be set on a Pipeline with the method ``set_hyperparams``. + def get_children_values(self, root_name: str, from_root=True) -> 'RecursiveDict': + remainders = RecursiveDict() + for name, hparams in self.items(): + if self.separator in name: + name_split = name.split(self.separator) + + if name_split[0] == root_name: + remainders[self.separator.join(name_split[1:])] = hparams + elif from_root: + remainders[name] = hparams - HyperparameterSamples are often the result of calling ``.rvs()`` on an HyperparameterSpace.""" + return remainders - def to_flat(self) -> 'HyperparameterSamples': + def to_flat(self) -> 'RecursiveDict': """ Will create an equivalent flat HyperparameterSamples. :return: an HyperparameterSamples like self, flattened. """ - return nested_dict_to_flat(self, dict_ctor=HyperparameterSamples) + return nested_dict_to_flat(self, dict_ctor=self.recursive_dict_constructor) - def to_nested_dict(self) -> 'HyperparameterSamples': + def to_nested_dict(self) -> 'RecursiveDict': """ Will create an equivalent nested dict HyperparameterSamples. :return: an HyperparameterSamples like self, as a nested dict. """ - return flat_to_nested_dict(self, dict_ctor=HyperparameterSamples) + return flat_to_nested_dict(self, dict_ctor=self.recursive_dict_constructor) def to_flat_as_dict_primitive(self) -> dict: """ @@ -167,13 +164,73 @@ def to_nested_dict_as_ordered_dict_primitive(self) -> OrderedDict: return flat_to_nested_dict(self, dict_ctor=OrderedDict) -class HyperparameterSpace(HyperparameterSamples): - """Wraps an hyperparameter nested dict or flat dict, and offer a few more functions to process +def nested_dict_to_flat(nested_dict, dict_ctor=OrderedDict): + """ + Convert a nested hyperparameter dictionary to a flat one. + + :param nested_dict: a nested hyperparameter dictionary. + :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. + :return: a flat hyperparameter dictionary. + """ + ret = dict_ctor() + for k, v in nested_dict.items(): + if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor): + _ret = nested_dict.nested_dict_to_flat(v, dict_ctor) + for key, val in _ret.items(): + ret[k + PARAMS_SPLIT_SEQ + key] = val + else: + ret[k] = v + return ret + + +def flat_to_nested_dict(flat_dict, dict_ctor=dict): + """ + Convert a flat hyperparameter dictionary to a nested one. + + :param flat_dict: a flat hyperparameter dictionary. + :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. + :return: a nested hyperparameter dictionary. + """ + pre_ret = dict_ctor() + ret = dict_ctor() + for k, v in flat_dict.items(): + k, _, key = k.partition(PARAMS_SPLIT_SEQ) + if len(key) > 0: + if k not in pre_ret.keys(): + pre_ret[k] = dict_ctor() + pre_ret[k][key] = v + else: + ret[k] = v + for k, v in pre_ret.items(): + ret[k] = flat_dict.flat_to_nested_dict(v, dict_ctor) + return ret + + +class HyperparameterSamples(RecursiveDict): + """ + Wraps an hyperparameter nested dict or flat dict, and offer a few more functions. + + This can be set on a Pipeline with the method ``set_hyperparams``. + + HyperparameterSamples are often the result of calling ``.rvs()`` on an HyperparameterSpace. + """ + + def __init__(self, *args, **kwds): + super().__init__(recursive_dict_constructor=HyperparameterSamples, *args, **kwds) + + +class HyperparameterSpace(RecursiveDict): + """ + Wraps an hyperparameter nested dict or flat dict, and offer a few more functions to process all contained HyperparameterDistribution. This can be set on a Pipeline with the method ``set_hyperparams_space``. - Calling ``.rvs()`` on an ``HyperparameterSpace`` results in ``HyperparameterSamples``.""" + Calling ``.rvs()`` on an ``HyperparameterSpace`` results in ``HyperparameterSamples``. + """ + + def __init__(self, *args, **kwds): + super().__init__(recursive_dict_constructor=HyperparameterSpace, *args, **kwds) def rvs(self) -> 'HyperparameterSamples': """ @@ -196,7 +253,6 @@ def nullify(self): new_items.append((k, v)) return HyperparameterSamples(new_items) - def narrow_space_from_best_guess( self, best_guesses: 'HyperparameterSpace', kept_space_ratio: float = 0.5 ) -> 'HyperparameterSpace': @@ -228,19 +284,3 @@ def unnarrow(self) -> 'HyperparameterSpace': v = v.unnarrow() new_items.append((k, v)) return HyperparameterSpace(new_items) - - def to_flat(self) -> 'HyperparameterSpace': - """ - Will create an equivalent flat HyperparameterSpace. - - :return: an HyperparameterSpace like self, flattened. - """ - return nested_dict_to_flat(self, dict_ctor=HyperparameterSpace) - - def to_nested_dict(self) -> 'HyperparameterSpace': - """ - Will create an equivalent nested dict HyperparameterSpace. - - :return: an HyperparameterSpace like self, as a nested dict. - """ - return flat_to_nested_dict(self, dict_ctor=HyperparameterSpace) diff --git a/neuraxle/steps/loop.py b/neuraxle/steps/loop.py index 711e5e3a..df479258 100644 --- a/neuraxle/steps/loop.py +++ b/neuraxle/steps/loop.py @@ -158,6 +158,15 @@ def __init__(self, wrapped: BaseStep, copy_op=copy.deepcopy, cache_folder_when_n self.copy_op = copy_op def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + """ + Apply a method to all of the cloned steps. + + :param method_name: method name that need to be called on all steps + :param step_name: current pipeline step name + :param kargs: any additional arguments to be passed to the method + :param kwargs: any additional positional arguments to be passed to the method + :return: accumulated results + """ results = MetaStepMixin.apply(self, method_name=method_name, step_name=step_name, *kargs, **kwargs) if len(self) > 0: for _, step in self: @@ -165,10 +174,23 @@ def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: return results def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Union[Dict, Iterable]: + """ + Apply method to the meta step and its wrapped step. + + :param method: method to call with self + :param step_name: step name to apply the method to + :param kargs: any additional arguments to be passed to the method + :param kwargs: any additional positional arguments to be passed to the method + :return: accumulated results + """ + # apply method on wrapped step before cloned steps. results = MetaStepMixin.apply_method(self, method=method, step_name=step_name, *kargs, **kwargs) + + # apply method on cloned steps if len(self) > 0: for _, step in self: results.update(step.apply_method(method=method, step_name=step_name, *kargs, **kwargs)) + return results def _will_process(self, data_container: DataContainer, context: ExecutionContext) -> ('BaseStep', DataContainer): diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index 974c0c8d..b28bee18 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -79,10 +79,23 @@ def transform(self, data_inputs): return self.wrapped_sklearn_predictor.transform(data_inputs) def _set_hyperparams(self, hyperparams: HyperparameterSamples, root=False) -> BaseStep: + """ + Set hyperparams for base step, and the wrapped sklearn_predictor. + + :param hyperparams: + :param root: + :return: + """ BaseStep._set_hyperparams(self, hyperparams) step_hyperparams, remainders = BaseStep._create_recursive_step_values_and_remainders(self, hyperparams, root) + + # flatten the step hyperparams, and set the wrapped sklearn predictor params self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(step_hyperparams).to_flat_as_dict_primitive()) - self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(HyperparameterSamples(remainders).to_flat()).to_flat_as_dict_primitive()) + + # there is remainders when we need to set params for steps that are inside sklearn.pipeline.Pipeline... + self.wrapped_sklearn_predictor.set_params( + **HyperparameterSamples(HyperparameterSamples(remainders).to_flat()).to_flat_as_dict_primitive()) + return self def get_hyperparams(self): From 22431bfd2bded63124b53c363ca941a713b10717 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Thu, 7 May 2020 16:39:12 -0400 Subject: [PATCH 13/48] Recursive Dict Refactor Steps With Children Mixin --- neuraxle/base.py | 295 +++++++++++++----------------- neuraxle/steps/loop.py | 41 +---- testing/test_apply_to_children.py | 11 ++ 3 files changed, 147 insertions(+), 200 deletions(-) create mode 100644 testing/test_apply_to_children.py diff --git a/neuraxle/base.py b/neuraxle/base.py index 638377df..17418850 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1645,7 +1645,124 @@ def _sklearn_to_neuraxle_step(step) -> BaseStep: return step -class MetaStepMixin: +class StepWithChildrenMixin: + """ + Mixin to add behavior to the steps that have children (sub steps). + + .. seealso:: + :class:`~neuraxle.base.MetaStepMixin`, + :class:`~neuraxle.base.TruncableSteps`, + :class:`~neuraxle.base.TruncableSteps` + """ + def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + """ + Apply method to root, and children steps. + Split the root, and children values inside the arguments of type RecursiveDict. + + :param method_name: + :param step_name: + :param kargs: + :param kwargs: + :return: + """ + root_kwargs, children_kwargs = self._get_root_and_children_kwargs(**kwargs) + root_kargs, children_kargs = self._get_root_and_children_kargs(*kargs) + + results = BaseStep.apply(self, method_name=method_name, step_name=step_name, *root_kargs, **root_kwargs) + + step_name = self._get_step_name_for_children(step_name) + for children in self.get_children(): + children_results = children.apply(method_name=method_name, step_name=step_name, *children_kargs, **children_kwargs) + results.update(children_results) + + return results + + def _get_step_name_for_children(self, step_name): + """ + Return the pipeline step name (path separated by __). + + :param step_name: current accumulated pipelien step name or path + :return: + """ + if step_name is not None: + step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) + else: + step_name = self.name + return step_name + + def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Union[Dict, Iterable]: + """ + Apply method to root, and children steps. + Split the root, and children values inside the arguments of type RecursiveDict. + + :param method: method to call with self + :param step_name: step name to apply the method to + :param children_only: apply method to children only + :param kargs: any additional arguments to be passed to the method + :param kwargs: any additional positional arguments to be passed to the method + :return: accumulated results + """ + root_kwargs, children_kwargs = self._get_root_and_children_kwargs(**kwargs) + root_kargs, children_kargs = self._get_root_and_children_kargs(*kargs) + + results = BaseStep.apply_method(self, method=method, step_name=step_name, *root_kargs, **root_kwargs) + + step_name = self._get_step_name_for_children(step_name) + for children in self.get_children(): + children_results = children.apply_method(method=method, step_name=step_name, *children_kargs, **children_kwargs) + results.update(children_results) + + return results + + def _get_root_and_children_kargs(self, *kargs): + """ + Split root, and children values inside *kargs. + + :param kargs: kargs values + :return: + """ + root_kargs = list() + children_kargs = list() + for arg in kargs: + if isinstance(arg, RecursiveDict): + root_kargs.append(arg.get_root_values(self.name)) + children_kargs.append(arg.get_children_values(self.name)) + else: + root_kargs.append(arg) + children_kargs.append(arg) + + return root_kargs, children_kargs + + def _get_root_and_children_kwargs(self, **kwargs): + """ + Split root, and children values inside *kwargs. + + :param kwargs: kwargs values + :return: + """ + root_kwargs = dict() + children_kwargs = dict() + for key in kwargs: + if isinstance(kwargs[key], RecursiveDict): + root_kwargs[key] = kwargs[key].get_root_values(self.name) + children_kwargs[key] = kwargs[key].get_children_values(self.name) + else: + root_kwargs[key] = kwargs[key] + children_kwargs[key] = kwargs[key] + + return root_kwargs, children_kwargs + + @abstractmethod + def get_children(self) -> List[BaseStep]: + """ + Get the list of all the children for that step. + + :return: + """ + pass + + +class MetaStepMixin(StepWithChildrenMixin): """ A class to represent a step that wraps another step. It can be used for many things. @@ -1837,56 +1954,13 @@ def resume(self, data_container: DataContainer, context: ExecutionContext): data_container = self._did_process(data_container, context) return data_container - def apply(self, method_name: str, step_name=None, children_only=False, *kargs, **kwargs) -> Dict: + def get_children(self) -> List[BaseStep]: """ - Apply the method name to the meta step and its wrapped step. + Get the list of all the children for that step. - :param method_name: method name that need to be called on all steps - :param step_name: step name to apply the method to - :param children_only: apply method to children only - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results - """ - results = BaseStep.apply(self, method_name=method_name, step_name=step_name, children_only=children_only, - *kargs, **kwargs) - - if step_name is not None: - step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) - else: - step_name = self.name - - if self.wrapped is not None: - wrapped_results = self.wrapped.apply(method_name=method_name, step_name=step_name, *kargs, **kwargs) - results.update(wrapped_results) - - return results - - def apply_method(self, method: Callable, step_name=None, children_only=False, *kargs, **kwargs) -> Union[ - Dict, Iterable]: - """ - Apply method to the meta step and its wrapped step. - - :param method: method to call with self - :param step_name: step name to apply the method to - :param children_only: apply method to children only - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results + :return: """ - results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, - **kwargs) - - if step_name is not None: - step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) - else: - step_name = self.name - - if self.wrapped is not None: - wrapped_results = self.wrapped.apply_method(method=method, step_name=step_name, *kargs, **kwargs) - results.update(wrapped_results) - - return results + return [self.wrapped] def get_step_by_name(self, name): if self.wrapped.name == name: @@ -2138,7 +2212,7 @@ def load_step(self, step: 'TruncableSteps', context: ExecutionContext) -> 'Trunc return step -class TruncableSteps(BaseStep, ABC): +class TruncableSteps(StepWithChildrenMixin, BaseStep, ABC): """ Step that contains multiple steps. :class:`Pipeline` inherits form this class. It is possible to truncate this step * :func:`~neuraxle.base.TruncableSteps.__getitem__` @@ -2158,6 +2232,7 @@ def __init__( hyperparams_space: HyperparameterSpace = dict() ): BaseStep.__init__(self, hyperparams=hyperparams, hyperparams_space=hyperparams_space) + StepWithChildrenMixin.__init__(self) self.set_steps(steps_as_tuple) self.set_savers([TruncableJoblibStepSaver()] + self.savers) @@ -2234,54 +2309,13 @@ def teardown(self) -> 'BaseStep': return self - def apply(self, method_name: str, step_name=None, children_only: bool = False, *kargs, **kwargs) -> Dict: - """ - Apply the method name to the pipeline step and all of its children. - - :param method_name: method name that need to be called on all steps - :param step_name: current pipeline step name - :param children_only: apply method only to the sub steps - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results - """ - results = BaseStep.apply(self, method_name, step_name=step_name, children_only=children_only, *kargs, **kwargs) - - if step_name is not None: - step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) - else: - step_name = self.name - - for step in self.values(): - sub_step_results = step.apply(method_name=method_name, step_name=step_name, *kargs, **kwargs) - results.update(sub_step_results) - - return results - - def apply_method(self, method: Callable, step_name=None, children_only=False, *kargs, **kwargs) -> Dict: + def get_children(self) -> List[BaseStep]: """ - Apply a method to the pipeline step and all of its children. + Get the list of sub step inside the step with children. - :param method: method to call with self - :param step_name: current pipeline step name - :param children_only: apply method only to the sub steps - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results + :return: children steps """ - results = BaseStep.apply_method(self, method=method, step_name=step_name, children_only=children_only, *kargs, - **kwargs) - - if step_name is not None: - step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) - else: - step_name = self.name - - for step in self.values(): - sub_step_results = step.apply_method(method=method, step_name=step_name, *kargs, **kwargs) - results.update(sub_step_results) - - return results + return self.values() def get_step_by_name(self, name): for step in self.values(): @@ -2992,76 +3026,3 @@ def load(self, context: ExecutionContext, full_dump=True) -> BaseStep: context.pop() return loaded_self.load(context, full_dump) - - -class StepsWithChildrenMixin: - def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: - """ - Apply method to root, and children steps. - Split the root, and children values inside the arguments of type RecursiveDict. - - :param method_name: - :param step_name: - :param kargs: - :param kwargs: - :return: - """ - root_kwargs, children_kwargs = self._get_root_and_children_kwargs(**kwargs) - root_kargs, children_kargs = self._get_root_and_children_kargs(*kargs) - - results = BaseStep.apply(self, method_name=method_name, step_name=step_name, *root_kargs, **root_kwargs) - children_results = self.apply_to_children(self, method_name=method_name, step_name=step_name, *root_kargs, **root_kwargs) - results.update(children_results) - - return results - - def _get_root_and_children_kargs(self, *kargs): - """ - Split root, and children values inside *kargs. - - :param kargs: kargs values - :return: - """ - root_kargs = list() - children_kargs = list() - for arg in kargs: - if isinstance(arg, RecursiveDict): - root_kargs.append(arg.get_root_values(self.name)) - children_kargs.append(arg.get_children_values(self.name)) - else: - root_kargs.append(arg) - children_kargs.append(arg) - - return root_kargs, children_kargs - - def _get_root_and_children_kwargs(self, **kwargs): - """ - Split root, and children values inside *kwargs. - - :param kwargs: kwargs values - :return: - """ - root_kwargs = dict() - children_kwargs = dict() - for key in kwargs: - if isinstance(kwargs[key], RecursiveDict): - root_kwargs[key] = kwargs[key].get_root_values(self.name) - children_kwargs[key] = kwargs[key].get_children_values(self.name) - else: - root_kwargs[key] = kwargs[key] - children_kwargs[key] = kwargs[key] - - return root_kwargs, children_kwargs - - @abstractmethod - def apply_to_children(self, method_name: str, step_name=None, *kargs, **kwargs): - """ - Base method to apply a method to all sub steps. - - :param method_name: method name that need to be called on all steps - :param step_name: current pipeline step name - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results - """ - pass diff --git a/neuraxle/steps/loop.py b/neuraxle/steps/loop.py index df479258..b5705149 100644 --- a/neuraxle/steps/loop.py +++ b/neuraxle/steps/loop.py @@ -28,7 +28,7 @@ import numpy as np from neuraxle.base import MetaStepMixin, BaseStep, DataContainer, ExecutionContext, ResumableStepMixin, \ - ForceHandleOnlyMixin, ForceHandleMixin, TruncableJoblibStepSaver, NamedTupleList + ForceHandleOnlyMixin, ForceHandleMixin, TruncableJoblibStepSaver, NamedTupleList, StepWithChildrenMixin from neuraxle.data_container import ListDataContainer from neuraxle.hyperparams.space import HyperparameterSamples, HyperparameterSpace @@ -157,41 +157,16 @@ def __init__(self, wrapped: BaseStep, copy_op=copy.deepcopy, cache_folder_when_n self.steps_as_tuple: List[NamedTupleList] = [] self.copy_op = copy_op - def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + def get_children(self) -> List[BaseStep]: """ - Apply a method to all of the cloned steps. + Get the list of all the children for that step. - :param method_name: method name that need to be called on all steps - :param step_name: current pipeline step name - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results + :return: list of children """ - results = MetaStepMixin.apply(self, method_name=method_name, step_name=step_name, *kargs, **kwargs) - if len(self) > 0: - for _, step in self: - results.update(step.apply(method_name=method_name, step_name=step_name, *kargs, **kwargs)) - return results - - def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Union[Dict, Iterable]: - """ - Apply method to the meta step and its wrapped step. - - :param method: method to call with self - :param step_name: step name to apply the method to - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results - """ - # apply method on wrapped step before cloned steps. - results = MetaStepMixin.apply_method(self, method=method, step_name=step_name, *kargs, **kwargs) - - # apply method on cloned steps - if len(self) > 0: - for _, step in self: - results.update(step.apply_method(method=method, step_name=step_name, *kargs, **kwargs)) - - return results + children: List[BaseStep] = MetaStepMixin.get_children(self) + cloned_children = self.steps_as_tuple + children.extend(cloned_children) + return children def _will_process(self, data_container: DataContainer, context: ExecutionContext) -> ('BaseStep', DataContainer): data_container, context = BaseStep._will_process(self, data_container, context) diff --git a/testing/test_apply_to_children.py b/testing/test_apply_to_children.py new file mode 100644 index 00000000..c2674ece --- /dev/null +++ b/testing/test_apply_to_children.py @@ -0,0 +1,11 @@ +from neuraxle.base import Identity +from neuraxle.pipeline import Pipeline + + +def test_apply_to_children(): + p = Pipeline([ + Identity(), + Identity() + ]) + + p.apply() From cce83626937530ebad0d52f7f37865d0bfe60d9a Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Thu, 7 May 2020 22:21:03 -0400 Subject: [PATCH 14/48] Refactor recursive arguments in apply methods with _RecursiveArguments --- neuraxle/base.py | 369 ++++++++++++++-------------------- neuraxle/hyperparams/space.py | 26 +-- neuraxle/steps/loop.py | 2 +- neuraxle/steps/sklearn.py | 13 +- 4 files changed, 167 insertions(+), 243 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 17418850..bec66b99 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -41,8 +41,6 @@ from neuraxle.data_container import DataContainer from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples, RecursiveDict -RECURSIVE_STEP_SEPARATOR = '__' - DEFAULT_CACHE_FOLDER = os.path.join(os.getcwd(), 'cache') @@ -644,6 +642,9 @@ def invalidate(self) -> 'BaseStep': :return: self """ + return self._invalidate() + + def _invalidate(self): self.is_invalidated = True return self @@ -668,8 +669,10 @@ def set_train(self, is_train: bool = True): .. seealso:: :func:`BaseStep.set_train` """ + return self._set_train(is_train) + + def _set_train(self, is_train): self.is_train = is_train - self.apply(method_name='set_train', is_train=is_train, children_only=True) return self def set_name(self, name: str): @@ -737,19 +740,12 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ - self._set_hyperparams(hyperparams=hyperparams, root=True) - return self + return self._set_hyperparams(hyperparams) - def _set_hyperparams(self, hyperparams: HyperparameterSamples, root=False) -> 'BaseStep': + def _set_hyperparams(self, hyperparams): self.invalidate() - hyperparams = HyperparameterSamples(hyperparams).to_flat() - - step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams, root) - self.hyperparams = HyperparameterSamples(step_hyperparams) if len(step_hyperparams) > 0 else self.hyperparams - - self.apply(method_name='_set_hyperparams', hyperparams=remainders, children_only=True) - + self.hyperparams = HyperparameterSamples(hyperparams) if len(hyperparams) > 0 else self.hyperparams return self def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': @@ -780,38 +776,12 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ - self._update_hyperparams(hyperparams=hyperparams, root=True) - return self - - def _update_hyperparams(self, hyperparams: HyperparameterSamples, root=False) -> 'BaseStep': - hyperparams = HyperparameterSamples(hyperparams).to_flat() - - step_hyperparams, remainders = self._create_recursive_step_values_and_remainders(hyperparams, root) - self.hyperparams.update(step_hyperparams) - - self.apply(method_name='_update_hyperparams', hyperparams=remainders, children_only=True) + self._update_hyperparams(hyperparams) return self - def _create_recursive_step_values_and_remainders(self, hyperparams, root=False): - remainders = dict() - step_values_dict = dict() - - for name, hparams in hyperparams.items(): - if RECURSIVE_STEP_SEPARATOR in name: - name_split = name.split(RECURSIVE_STEP_SEPARATOR) - - if str(name_split[-2]) == self.name and len(name_split) == 2: - step_values_dict[name_split[-1]] = hparams - elif name_split[0] == self.name: - remainders[RECURSIVE_STEP_SEPARATOR.join(name_split[1:])] = hparams - elif root: - remainders[name] = hparams - - else: - step_values_dict[name] = hparams - - return step_values_dict, remainders + def _update_hyperparams(self, hyperparams: HyperparameterSamples): + self.hyperparams.update(HyperparameterSamples(hyperparams).to_flat()) def get_hyperparams(self) -> HyperparameterSamples: """ @@ -822,16 +792,10 @@ def get_hyperparams(self) -> HyperparameterSamples: .. seealso:: * :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ - results = self.apply(method_name='get_hyperparams', children_only=True) - results = HyperparameterSamples({ - RECURSIVE_STEP_SEPARATOR.join(key.split(RECURSIVE_STEP_SEPARATOR)[1:]): value.to_flat() - for key, value in results.items() - }).to_flat() + return self._get_hyperparams() - return HyperparameterSamples({ - **self.hyperparams.to_flat_as_dict_primitive(), - **results - }) + def _get_hyperparams(self): + return HyperparameterSamples(self.hyperparams.to_flat_as_dict_primitive()) def set_params(self, **params) -> 'BaseStep': """ @@ -890,18 +854,9 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ - self._set_hyperparams_space(hyperparams_space=hyperparams_space, root=True) - return self - - def _set_hyperparams_space(self, hyperparams_space: HyperparameterSpace, root=False) -> 'BaseStep': self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() - - step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space, root) - self.hyperparams_space = HyperparameterSpace(step_hyperparams_space) if len( - step_hyperparams_space) > 0 else self.hyperparams_space - - self.apply(method_name='_set_hyperparams_space', hyperparams_space=remainders, children_only=True) + self.hyperparams_space = HyperparameterSpace(hyperparams_space) if len(hyperparams_space) > 0 else self.hyperparams_space return self @@ -933,16 +888,9 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - self._update_hyperparams_space(hyperparams_space=hyperparams_space, root=True) - return self - - def _update_hyperparams_space(self, hyperparams_space: HyperparameterSpace, root=False) -> 'BaseStep': + self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() - - step_hyperparams_space, remainders = self._create_recursive_step_values_and_remainders(hyperparams_space, root) - self.hyperparams_space.update(HyperparameterSpace(step_hyperparams_space).to_flat()) - - self.apply(method_name='_update_hyperparams_space', hyperparams_space=remainders, children_only=True) + self.hyperparams_space.update(HyperparameterSpace(hyperparams_space).to_flat()) return self @@ -963,16 +911,7 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ - results = self.apply(method_name='get_hyperparams_space', children_only=True) - results = HyperparameterSpace({ - RECURSIVE_STEP_SEPARATOR.join(key.split(RECURSIVE_STEP_SEPARATOR)[1:]): value.to_flat() - for key, value in results.items() - }).to_flat() - - return HyperparameterSpace({ - **self.hyperparams_space, - **results - }) + return HyperparameterSpace(self.hyperparams_space.to_flat_as_dict_primitive()) def handle_inverse_transform(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: """ @@ -993,58 +932,38 @@ def handle_inverse_transform(self, data_container: DataContainer, context: Execu return data_container - def _inverse_transform_data_container( - self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: + def _inverse_transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: processed_outputs = self.inverse_transform(data_container.data_inputs) data_container.set_data_inputs(processed_outputs) return data_container - def apply_method(self, method: Callable, step_name=None, children_only: bool = False, *kargs, **kwargs) -> Dict: + def apply(self, method: str, *kargs, **kwargs) -> Dict: """ Apply a method to a step and its children. - :param method: method to call with self - :param step_name: current pipeline step name - :param children_only: apply method only to the sub steps + :param method: method name that need to be called on all steps :param kargs: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - if children_only: - return {} - - if step_name is not None: - step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) - else: - step_name = self.name + results = self._apply(method=method, arguments=_RecursiveArguments(context=self.name, *kargs, **kwargs)) + return results.to_flat_as_dict_primitive() - return { - step_name: method(self, *kargs, **kwargs) - } - - def apply(self, method_name: str, step_name=None, children_only: bool = False, *kargs, **kwargs) -> Dict: + def _apply(self, method: str, arguments: '_RecursiveArguments') -> RecursiveDict: """ Apply a method to a step and its children. - :param method_name: method name that need to be called on all steps - :param step_name: current pipeline step name - :param children_only: apply method only to the sub steps - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results + :param method: method name that need to be called on all steps + :param arguments: current pipeline step name + :return: recursive dict containing the results """ - results = {} - if children_only: - return results - - if step_name is not None: - step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) - else: - step_name = self.name - - if hasattr(self, method_name) and callable(getattr(self, method_name)): - results[step_name] = getattr(self, method_name)(*kargs, **kwargs) + results = RecursiveDict() + kargs, kwargs = arguments[self.name] + if isinstance(method, Callable): + results[arguments.context] = method(self, *kargs, **kwargs) + elif hasattr(self, method) and callable(getattr(self, method)): + results[arguments.context] = getattr(self, method)(*kargs, **kwargs) return results @@ -1645,7 +1564,67 @@ def _sklearn_to_neuraxle_step(step) -> BaseStep: return step -class StepWithChildrenMixin: +class _RecursiveArguments: + def __init__(self, context: str, *kargs, **kwargs): + self.context = context + self.kargs = kargs + self.kwargs = kwargs + + def push(self, step_name: str): + """ + Push step name to context. + + :param step_name: step name + :return: + """ + self.context = "{}{}{}".format(self.context, RecursiveDict.DEFAULT_SEPARATOR, step_name) + return step_name + + def __getitem__(self, item: str): + """ + Return arguments for the given path. + + :param item: root name + :return: + """ + if item is None: + # ra[None] gives the root level and purges the childs in the returned new copy of RA. + pass + + arguments = self._get_arguments_for(item) + keyword_arguments = self._get_keyword_arguments_for(item) + + return arguments, keyword_arguments + + def _get_keyword_arguments_for(self, item): + keyword_arguments = dict() + for key in self.kwargs: + if isinstance(self.kwargs[key], RecursiveDict): + keyword_arguments[key] = self.kwargs[key][item] + else: + keyword_arguments[key] = keyword_arguments[key] + return keyword_arguments + + def _get_arguments_for(self, item): + arguments = list() + for arg in self.kargs: + if isinstance(arg, RecursiveDict): + arguments.append(arg[item]) + return arguments + +# def apply(func, ra=None, *, **): +# ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *, **) +# result: RecursiveDict = BaseStep.apply(self, func, ra[None]) # ra[None] gives the root level and purges the childs in the returned new copy of RA. +# result: RecursiveDict = self.apply_childrends(self, result, func, ra) +# return result +# +# def apply_childrends(self, result: RecursiveDict, func: Union, ra: _RecursiveArguments) -> RecursiveDict: +# for children in self.get_kids(): +# result[children.get_name()] = children.apply(func, ra[children.get_name()]) +# return result + + +class _HasChildrenMixin: """ Mixin to add behavior to the steps that have children (sub steps). @@ -1654,115 +1633,93 @@ class StepWithChildrenMixin: :class:`~neuraxle.base.TruncableSteps`, :class:`~neuraxle.base.TruncableSteps` """ - def apply(self, method_name: str, step_name=None, *kargs, **kwargs) -> Dict: + def _apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *args, **kwargs) -> RecursiveDict: """ Apply method to root, and children steps. Split the root, and children values inside the arguments of type RecursiveDict. - :param method_name: - :param step_name: - :param kargs: - :param kwargs: + :param method: str or callable function to apply + :param ra: recursive arguments :return: """ - root_kwargs, children_kwargs = self._get_root_and_children_kwargs(**kwargs) - root_kargs, children_kargs = self._get_root_and_children_kargs(*kargs) - - results = BaseStep.apply(self, method_name=method_name, step_name=step_name, *root_kargs, **root_kwargs) - - step_name = self._get_step_name_for_children(step_name) + ra: _RecursiveArguments = _RecursiveArguments(arguments=ra, *args, **kwargs) + results: RecursiveDict = BaseStep.apply(self, method=method, ra=ra[None]) for children in self.get_children(): - children_results = children.apply(method_name=method_name, step_name=step_name, *children_kargs, **children_kwargs) - results.update(children_results) + results[children.get_name()] = children._apply(method=method, arguments=ra) return results - def _get_step_name_for_children(self, step_name): + @abstractmethod + def get_children(self) -> List[BaseStep]: """ - Return the pipeline step name (path separated by __). + Get the list of all the children for that step. - :param step_name: current accumulated pipelien step name or path :return: """ - if step_name is not None: - step_name = "{}{}{}".format(step_name, RECURSIVE_STEP_SEPARATOR, self.name) - else: - step_name = self.name - return step_name + pass - def apply_method(self, method: Callable, step_name=None, *kargs, **kwargs) -> Union[Dict, Iterable]: + def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': """ - Apply method to root, and children steps. - Split the root, and children values inside the arguments of type RecursiveDict. + Update hyperparams recursively by applying :func:`~BaseStep.update_hyperparams_space` to all steps. - :param method: method to call with self - :param step_name: step name to apply the method to - :param children_only: apply method to children only - :param kargs: any additional arguments to be passed to the method - :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results + .. seealso:: + :func:`~BaseStep.apply`, + :func:`~BaseStep.update_hyperparams`, + :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - root_kwargs, children_kwargs = self._get_root_and_children_kwargs(**kwargs) - root_kargs, children_kargs = self._get_root_and_children_kargs(*kargs) - - results = BaseStep.apply_method(self, method=method, step_name=step_name, *root_kargs, **root_kwargs) - - step_name = self._get_step_name_for_children(step_name) - for children in self.get_children(): - children_results = children.apply_method(method=method, step_name=step_name, *children_kargs, **children_kwargs) - results.update(children_results) - - return results + self._apply(method='_update_hyperparams', ra=_RecursiveArguments(context=self.name, hyperparams=hyperparams)) + return self - def _get_root_and_children_kargs(self, *kargs): + def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': """ - Split root, and children values inside *kargs. + Update hyperparams space recursively by applying :func:`~BaseStep.update_hyperparams_space` to all steps. - :param kargs: kargs values - :return: + .. seealso:: + :func:`~BaseStep.apply`, + :func:`~BaseStep.update_hyperparams_space`, + :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - root_kargs = list() - children_kargs = list() - for arg in kargs: - if isinstance(arg, RecursiveDict): - root_kargs.append(arg.get_root_values(self.name)) - children_kargs.append(arg.get_children_values(self.name)) - else: - root_kargs.append(arg) - children_kargs.append(arg) + self._apply(method='_update_hyperparams_space', ra=_RecursiveArguments(context=self.name, hyperparams_space=hyperparams_space)) + return self - return root_kargs, children_kargs + def get_hyperparams_space(self) -> HyperparameterSpace: + """ + Get all the pipeline hyperparams by applying :func:`~BaseStep.get_hyperparams_space` to all steps. - def _get_root_and_children_kwargs(self, **kwargs): + .. seealso:: + :func:`~BaseStep.apply`, + :func:`~BaseStep.get_hyperparams_space`, + :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - Split root, and children values inside *kwargs. + return HyperparameterSpace(**self._apply(method='_get_hyperparams_space', ra=_RecursiveArguments(context=self.name))) - :param kwargs: kwargs values - :return: + def get_hyperparams(self) -> HyperparameterSamples: """ - root_kwargs = dict() - children_kwargs = dict() - for key in kwargs: - if isinstance(kwargs[key], RecursiveDict): - root_kwargs[key] = kwargs[key].get_root_values(self.name) - children_kwargs[key] = kwargs[key].get_children_values(self.name) - else: - root_kwargs[key] = kwargs[key] - children_kwargs[key] = kwargs[key] + Get all the pipeline hyperparams by applying :func:`~BaseStep.get_hyperparams` to all steps. - return root_kwargs, children_kwargs + .. seealso:: + :func:`~BaseStep.apply`, + :func:`~BaseStep.get_hyperparams`, + :class:`~neuraxle.hyperparams.space.HyperparameterSpace` + """ + return HyperparameterSamples(**self._apply(method='_get_hyperparams', ra=_RecursiveArguments(context=self.name))) - @abstractmethod - def get_children(self) -> List[BaseStep]: + def set_train(self, is_train: bool = True) -> BaseStep: """ - Get the list of all the children for that step. + This method overrides the method of BaseStep to also consider the wrapped step as well as self. + Set pipeline step mode to train or test. + :param is_train: is training mode or not :return: + + .. seealso:: + :func:`BaseStep.set_train` """ - pass + self._apply(method='_set_train', ra=_RecursiveArguments(context=self.name, is_train=is_train)) + return self -class MetaStepMixin(StepWithChildrenMixin): +class MetaStepMixin(_HasChildrenMixin): """ A class to represent a step that wraps another step. It can be used for many things. @@ -2212,7 +2169,7 @@ def load_step(self, step: 'TruncableSteps', context: ExecutionContext) -> 'Trunc return step -class TruncableSteps(StepWithChildrenMixin, BaseStep, ABC): +class TruncableSteps(_HasChildrenMixin, BaseStep, ABC): """ Step that contains multiple steps. :class:`Pipeline` inherits form this class. It is possible to truncate this step * :func:`~neuraxle.base.TruncableSteps.__getitem__` @@ -2232,7 +2189,7 @@ def __init__( hyperparams_space: HyperparameterSpace = dict() ): BaseStep.__init__(self, hyperparams=hyperparams, hyperparams_space=hyperparams_space) - StepWithChildrenMixin.__init__(self) + _HasChildrenMixin.__init__(self) self.set_steps(steps_as_tuple) self.set_savers([TruncableJoblibStepSaver()] + self.savers) @@ -2711,34 +2668,6 @@ def ends_with(self, step_type: type): """ return isinstance(self[-1], step_type) - def set_train(self, is_train: bool = True) -> 'BaseStep': - """ - Set pipeline step mode to train or test. - - In the pipeline steps functions, you can add a simple if statement to direct to the right implementation: - - .. code-block:: python - - def transform(self, data_inputs): - if self.is_train: - self.transform_train_(data_inputs) - else: - self.transform_test_(data_inputs) - - def fit_transform(self, data_inputs, expected_outputs): - if self.is_train: - self.fit_transform_train_(data_inputs, expected_outputs) - else: - self.fit_transform_test_(data_inputs, expected_outputs) - - :param is_train: if the step is in train mode (True) or test mode (False) - :return: self - """ - self.is_train = is_train - for _, step in self.items(): - step.set_train(is_train) - return self - def __repr__(self): output = self.__class__.__name__ + '\n' \ diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index 9ff92741..bc802ca0 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -89,19 +89,6 @@ def __init__(self, separator=None, recursive_dict_constructor=None, *args, **kwd self.separator = separator self.recursive_dict_constructor = recursive_dict_constructor - def get_root_values(self, root_name: str): - root_dict_values = dict() - - for name, hparams in self.items(): - if self.separator in name: - name_split = name.split(self.separator) - if str(name_split[-2]) == root_name and len(name_split) == 2: - root_dict_values[name_split[-1]] = hparams - else: - root_dict_values[name] = hparams - - return root_dict_values - def get_children_values(self, root_name: str, from_root=True) -> 'RecursiveDict': remainders = RecursiveDict() for name, hparams in self.items(): @@ -115,6 +102,19 @@ def get_children_values(self, root_name: str, from_root=True) -> 'RecursiveDict' return remainders + def __getitem__(self, item: str): + root_dict_values = dict() + + for name, hparams in self.items(): + if self.separator in name: + name_split = name.split(self.separator) + if str(name_split[-2]) == item and len(name_split) == 2: + root_dict_values[name_split[-1]] = hparams + else: + root_dict_values[name] = hparams + + return root_dict_values + def to_flat(self) -> 'RecursiveDict': """ Will create an equivalent flat HyperparameterSamples. diff --git a/neuraxle/steps/loop.py b/neuraxle/steps/loop.py index b5705149..5c9927bb 100644 --- a/neuraxle/steps/loop.py +++ b/neuraxle/steps/loop.py @@ -28,7 +28,7 @@ import numpy as np from neuraxle.base import MetaStepMixin, BaseStep, DataContainer, ExecutionContext, ResumableStepMixin, \ - ForceHandleOnlyMixin, ForceHandleMixin, TruncableJoblibStepSaver, NamedTupleList, StepWithChildrenMixin + ForceHandleOnlyMixin, ForceHandleMixin, TruncableJoblibStepSaver, NamedTupleList, _HasChildrenMixin from neuraxle.data_container import ListDataContainer from neuraxle.hyperparams.space import HyperparameterSamples, HyperparameterSpace diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index b28bee18..93dede3a 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -78,23 +78,18 @@ def transform(self, data_inputs): return self.wrapped_sklearn_predictor.predict(data_inputs) return self.wrapped_sklearn_predictor.transform(data_inputs) - def _set_hyperparams(self, hyperparams: HyperparameterSamples, root=False) -> BaseStep: + def _set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: """ Set hyperparams for base step, and the wrapped sklearn_predictor. :param hyperparams: - :param root: - :return: + :return: self """ - BaseStep._set_hyperparams(self, hyperparams) - step_hyperparams, remainders = BaseStep._create_recursive_step_values_and_remainders(self, hyperparams, root) - # flatten the step hyperparams, and set the wrapped sklearn predictor params - self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(step_hyperparams).to_flat_as_dict_primitive()) + self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(hyperparams).to_flat_as_dict_primitive()) # there is remainders when we need to set params for steps that are inside sklearn.pipeline.Pipeline... - self.wrapped_sklearn_predictor.set_params( - **HyperparameterSamples(HyperparameterSamples(remainders).to_flat()).to_flat_as_dict_primitive()) + self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(HyperparameterSamples(hyperparams).to_flat()).to_flat_as_dict_primitive()) return self From 9d71b76905a926cb53d890800c58315a4938255f Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Fri, 8 May 2020 16:29:16 -0400 Subject: [PATCH 15/48] Refactor recursive arguments in apply methods with _RecursiveArguments --- neuraxle/base.py | 115 +++++++++++++-------------- neuraxle/hyperparams/space.py | 23 +++--- testing/test_recursive_arguments.py | 118 ++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 68 deletions(-) create mode 100644 testing/test_recursive_arguments.py diff --git a/neuraxle/base.py b/neuraxle/base.py index bec66b99..a9bf5c78 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -938,34 +938,35 @@ def _inverse_transform_data_container(self, data_container: DataContainer, conte return data_container - def apply(self, method: str, *kargs, **kwargs) -> Dict: + def apply(self, method: Union[str, Callable], ra: '_RecursiveArguments' = None, *args, **kwargs) -> RecursiveDict: """ Apply a method to a step and its children. :param method: method name that need to be called on all steps - :param kargs: any additional arguments to be passed to the method + :param ra: recursive arguments + :param args: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method :return: accumulated results """ - results = self._apply(method=method, arguments=_RecursiveArguments(context=self.name, *kargs, **kwargs)) - return results.to_flat_as_dict_primitive() + if ra is None: + ra = _RecursiveArguments(*args, **kwargs) + + return self._apply(method=method, ra=ra) - def _apply(self, method: str, arguments: '_RecursiveArguments') -> RecursiveDict: + def _apply(self, method: str, ra: '_RecursiveArguments') -> Any: """ Apply a method to a step and its children. :param method: method name that need to be called on all steps - :param arguments: current pipeline step name - :return: recursive dict containing the results + :param ra: current pipeline step name + :return: method outputs, or None if no method has been applied """ - results = RecursiveDict() - kargs, kwargs = arguments[self.name] if isinstance(method, Callable): - results[arguments.context] = method(self, *kargs, **kwargs) + return method(self, *ra.kargs, **ra.kwargs) elif hasattr(self, method) and callable(getattr(self, method)): - results[arguments.context] = getattr(self, method)(*kargs, **kwargs) + return getattr(self, method)(*ra.kargs, **ra.kwargs) - return results + return None def get_step_by_name(self, name): if self.name == name: @@ -1565,21 +1566,13 @@ def _sklearn_to_neuraxle_step(step) -> BaseStep: class _RecursiveArguments: - def __init__(self, context: str, *kargs, **kwargs): - self.context = context + def __init__(self, ra=None, *kargs, **kwargs): + if ra is not None: + kargs = ra.kargs + kwargs = ra.kwargs self.kargs = kargs self.kwargs = kwargs - def push(self, step_name: str): - """ - Push step name to context. - - :param step_name: step name - :return: - """ - self.context = "{}{}{}".format(self.context, RecursiveDict.DEFAULT_SEPARATOR, step_name) - return step_name - def __getitem__(self, item: str): """ Return arguments for the given path. @@ -1588,29 +1581,32 @@ def __getitem__(self, item: str): :return: """ if item is None: - # ra[None] gives the root level and purges the childs in the returned new copy of RA. - pass - - arguments = self._get_arguments_for(item) - keyword_arguments = self._get_keyword_arguments_for(item) - - return arguments, keyword_arguments - - def _get_keyword_arguments_for(self, item): - keyword_arguments = dict() - for key in self.kwargs: - if isinstance(self.kwargs[key], RecursiveDict): - keyword_arguments[key] = self.kwargs[key][item] - else: - keyword_arguments[key] = keyword_arguments[key] - return keyword_arguments + arguments = list() + keyword_arguments = dict() + for arg in self.kargs: + if isinstance(arg, RecursiveDict): + arguments.append(arg[item]) + for key in self.kwargs: + if isinstance(self.kwargs[key], RecursiveDict): + keyword_arguments[key] = self.kwargs[key][item] + else: + keyword_arguments[key] = keyword_arguments[key] + return _RecursiveArguments(*arguments, **keyword_arguments) + else: + arguments = list() + keyword_arguments = dict() + for arg in self.kargs: + if isinstance(arg, RecursiveDict): + arguments.append(arg[item]) + for key in self.kwargs: + if isinstance(self.kwargs[key], RecursiveDict): + keyword_arguments[key] = self.kwargs[key][item] + else: + keyword_arguments[key] = keyword_arguments[key] + return _RecursiveArguments(*arguments, **keyword_arguments) - def _get_arguments_for(self, item): - arguments = list() - for arg in self.kargs: - if isinstance(arg, RecursiveDict): - arguments.append(arg[item]) - return arguments + def __iter__(self): + return self.kwargs # def apply(func, ra=None, *, **): # ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *, **) @@ -1633,7 +1629,7 @@ class _HasChildrenMixin: :class:`~neuraxle.base.TruncableSteps`, :class:`~neuraxle.base.TruncableSteps` """ - def _apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *args, **kwargs) -> RecursiveDict: + def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *args, **kwargs) -> RecursiveDict: """ Apply method to root, and children steps. Split the root, and children values inside the arguments of type RecursiveDict. @@ -1642,13 +1638,18 @@ def _apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, * :param ra: recursive arguments :return: """ - ra: _RecursiveArguments = _RecursiveArguments(arguments=ra, *args, **kwargs) - results: RecursiveDict = BaseStep.apply(self, method=method, ra=ra[None]) - for children in self.get_children(): - results[children.get_name()] = children._apply(method=method, arguments=ra) + ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *args, **kwargs) + results: RecursiveDict = RecursiveDict() + results[self.get_name()] = BaseStep.apply(self, method=method, ra=ra[None]) + results: RecursiveDict = self._apply_childrends(results=results, method=method, ra=ra) return results + def _apply_childrends(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): + for children in self.get_children(): + results[children.get_name()] = children.apply(method=method, ra=ra) + return results + @abstractmethod def get_children(self) -> List[BaseStep]: """ @@ -1667,7 +1668,7 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - self._apply(method='_update_hyperparams', ra=_RecursiveArguments(context=self.name, hyperparams=hyperparams)) + self.apply(method='_update_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams)) return self def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': @@ -1679,7 +1680,7 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B :func:`~BaseStep.update_hyperparams_space`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - self._apply(method='_update_hyperparams_space', ra=_RecursiveArguments(context=self.name, hyperparams_space=hyperparams_space)) + self.apply(method='_update_hyperparams_space', ra=_RecursiveArguments(hyperparams_space=hyperparams_space)) return self def get_hyperparams_space(self) -> HyperparameterSpace: @@ -1691,7 +1692,7 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :func:`~BaseStep.get_hyperparams_space`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - return HyperparameterSpace(**self._apply(method='_get_hyperparams_space', ra=_RecursiveArguments(context=self.name))) + return HyperparameterSpace(**self.apply(method='_get_hyperparams_space', ra=_RecursiveArguments())) def get_hyperparams(self) -> HyperparameterSamples: """ @@ -1702,7 +1703,7 @@ def get_hyperparams(self) -> HyperparameterSamples: :func:`~BaseStep.get_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - return HyperparameterSamples(**self._apply(method='_get_hyperparams', ra=_RecursiveArguments(context=self.name))) + return HyperparameterSamples(**self.apply(method='_get_hyperparams', ra=_RecursiveArguments())) def set_train(self, is_train: bool = True) -> BaseStep: """ @@ -1715,7 +1716,7 @@ def set_train(self, is_train: bool = True) -> BaseStep: .. seealso:: :func:`BaseStep.set_train` """ - self._apply(method='_set_train', ra=_RecursiveArguments(context=self.name, is_train=is_train)) + self.apply(method='_set_train', ra=_RecursiveArguments(is_train=is_train)) return self @@ -2272,7 +2273,7 @@ def get_children(self) -> List[BaseStep]: :return: children steps """ - return self.values() + return list(self.values()) def get_step_by_name(self, name): for step in self.values(): diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index bc802ca0..ba4fb2cf 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -90,7 +90,7 @@ def __init__(self, separator=None, recursive_dict_constructor=None, *args, **kwd self.recursive_dict_constructor = recursive_dict_constructor def get_children_values(self, root_name: str, from_root=True) -> 'RecursiveDict': - remainders = RecursiveDict() + remainders = RecursiveDict(self.separator) for name, hparams in self.items(): if self.separator in name: name_split = name.split(self.separator) @@ -102,18 +102,19 @@ def get_children_values(self, root_name: str, from_root=True) -> 'RecursiveDict' return remainders - def __getitem__(self, item: str): - root_dict_values = dict() + def __getitem__(self, item: str = None): + item_values = dict() + + for name, values in self.items(): + if item is None and not self.separator in name: + item_values[name] = values - for name, hparams in self.items(): if self.separator in name: name_split = name.split(self.separator) if str(name_split[-2]) == item and len(name_split) == 2: - root_dict_values[name_split[-1]] = hparams - else: - root_dict_values[name] = hparams + item_values[name_split[-1]] = values - return root_dict_values + return item_values def to_flat(self) -> 'RecursiveDict': """ @@ -175,7 +176,7 @@ def nested_dict_to_flat(nested_dict, dict_ctor=OrderedDict): ret = dict_ctor() for k, v in nested_dict.items(): if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor): - _ret = nested_dict.nested_dict_to_flat(v, dict_ctor) + _ret = nested_dict_to_flat(v, dict_ctor) for key, val in _ret.items(): ret[k + PARAMS_SPLIT_SEQ + key] = val else: @@ -216,7 +217,7 @@ class HyperparameterSamples(RecursiveDict): """ def __init__(self, *args, **kwds): - super().__init__(recursive_dict_constructor=HyperparameterSamples, *args, **kwds) + super().__init__(RecursiveDict.DEFAULT_SEPARATOR, HyperparameterSamples, *args, **kwds) class HyperparameterSpace(RecursiveDict): @@ -230,7 +231,7 @@ class HyperparameterSpace(RecursiveDict): """ def __init__(self, *args, **kwds): - super().__init__(recursive_dict_constructor=HyperparameterSpace, *args, **kwds) + super().__init__(RecursiveDict.DEFAULT_SEPARATOR, HyperparameterSpace, *args, **kwds) def rvs(self) -> 'HyperparameterSamples': """ diff --git a/testing/test_recursive_arguments.py b/testing/test_recursive_arguments.py new file mode 100644 index 00000000..20bccf67 --- /dev/null +++ b/testing/test_recursive_arguments.py @@ -0,0 +1,118 @@ +from neuraxle.base import _RecursiveArguments, Identity +from neuraxle.hyperparams.space import HyperparameterSamples +from neuraxle.pipeline import Pipeline + + +def test_recursive_arguments_should_get_root_level(): + ra = _RecursiveArguments(hyperparams=HyperparameterSamples({ + 'hp0': 0, + 'hp1': 1, + 'pipeline__stepa__hp2': 2, + 'pipeline__stepb__hp3': 3 + })) + + root_ra = ra[None] + + root_ra.kargs == [] + root_ra.kwargs == {'hyperparams': HyperparameterSamples({ + 'hp0': 0, + 'hp1': 1 + })} + + +def test_recursive_arguments_should_get_one_recursive_level(): + ra = _RecursiveArguments(hyperparams=HyperparameterSamples({ + 'hp0': 0, + 'hp1': 1, + 'stepa__hp2': 2, + 'stepb__hp3': 3 + })) + + ra = ra['stepa'] + + ra.kargs == [] + ra.kwargs == {'hyperparams': HyperparameterSamples({ + 'stepa__hp2': 2 + })} + + +def test_recursive_arguments_should_have_copy_constructor(): + ra = _RecursiveArguments( + ra=_RecursiveArguments(hyperparams=HyperparameterSamples({ + 'hp0': 0, + 'hp1': 1 + })) + ) + + ra.kargs == [] + ra.kwargs == {'hyperparams': HyperparameterSamples({ + 'hp0': 0, + 'hp1': 1, + })} + + +def test_has_children_mixin_apply_should_apply_method_to_direct_childrends(): + p = Pipeline([ + ('a', Identity()), + ('b', Identity()), + Pipeline([ + ('c', Identity()), + ('d', Identity()) + ]), + ]) + + p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ + 'a__hp': 0, + 'b__hp': 1, + 'Pipeline__hp': 2, + })) + + assert p['a'].get_hyperparams()['hp'] == 0 + assert p['b'].get_hyperparams()['hp'] == 1 + assert p['Pipeline'].get_hyperparams()['hp'] == 2 + + +def test_has_children_mixin_apply_should_apply_method_to_recursive_childrends(): + p = Pipeline([ + ('a', Identity()), + ('b', Identity()), + Pipeline([ + ('c', Identity()), + ('d', Identity()) + ]), + ]) + + p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ + 'Pipeline__c__hp': 3, + 'Pipeline__d__hp': 4 + })) + + assert p['Pipeline']['c'].get_hyperparams()['hp'] == 3 + assert p['Pipeline']['d'].get_hyperparams()['hp'] == 4 + + +def test_has_children_mixin_apply_should_return_recursive_dict_to_direct_childrends(): + p = Pipeline([ + ('a', Identity().set_hyperparams(HyperparameterSamples({'hp': 0}))), + ('b', Identity().set_hyperparams(HyperparameterSamples({'hp': 1}))) + ]) + + results = p.apply('get_hyperparams', ra=None) + + assert results['a__hp'] == 0 + assert results['b__hp'] == 0 + + +def test_has_children_mixin_apply_should_return_recursive_dict_to_recursive_childrends(): + p = Pipeline([ + Pipeline([ + ('c', Identity().set_hyperparams(HyperparameterSamples({'hp': 3}))), + ('d', Identity().set_hyperparams(HyperparameterSamples({'hp': 4}))) + ]).set_hyperparams(HyperparameterSamples({'hp': 2})), + ]) + + results = p.apply('get_hyperparams', ra=None) + + assert results['Pipeline__hp'] == 2 + assert results['Pipeline__c__hp'] == 3 + assert results['Pipeline__d__hp'] == 4 From bf3694ce49c70cabacc20132a454e6c43afc6f46 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 10 May 2020 13:15:04 -0400 Subject: [PATCH 16/48] Refactor recursive arguments in apply methods with _RecursiveArguments --- neuraxle/base.py | 10 +++++++--- neuraxle/hyperparams/space.py | 19 +++---------------- testing/test_recursive_arguments.py | 12 +++++++----- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index a9bf5c78..ed7ae737 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1586,11 +1586,13 @@ def __getitem__(self, item: str): for arg in self.kargs: if isinstance(arg, RecursiveDict): arguments.append(arg[item]) + else: + arguments.append(arg) for key in self.kwargs: if isinstance(self.kwargs[key], RecursiveDict): keyword_arguments[key] = self.kwargs[key][item] else: - keyword_arguments[key] = keyword_arguments[key] + keyword_arguments[key] = self.kwargs[key] return _RecursiveArguments(*arguments, **keyword_arguments) else: arguments = list() @@ -1598,11 +1600,13 @@ def __getitem__(self, item: str): for arg in self.kargs: if isinstance(arg, RecursiveDict): arguments.append(arg[item]) + else: + arguments.append(arg) for key in self.kwargs: if isinstance(self.kwargs[key], RecursiveDict): keyword_arguments[key] = self.kwargs[key][item] else: - keyword_arguments[key] = keyword_arguments[key] + keyword_arguments[key] = self.kwargs[key] return _RecursiveArguments(*arguments, **keyword_arguments) def __iter__(self): @@ -1647,7 +1651,7 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a def _apply_childrends(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): for children in self.get_children(): - results[children.get_name()] = children.apply(method=method, ra=ra) + results[children.get_name()] = children.apply(method=method, ra=ra[children.get_name()]) return results @abstractmethod diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index ba4fb2cf..176f6b09 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -89,21 +89,8 @@ def __init__(self, separator=None, recursive_dict_constructor=None, *args, **kwd self.separator = separator self.recursive_dict_constructor = recursive_dict_constructor - def get_children_values(self, root_name: str, from_root=True) -> 'RecursiveDict': - remainders = RecursiveDict(self.separator) - for name, hparams in self.items(): - if self.separator in name: - name_split = name.split(self.separator) - - if name_split[0] == root_name: - remainders[self.separator.join(name_split[1:])] = hparams - elif from_root: - remainders[name] = hparams - - return remainders - def __getitem__(self, item: str = None): - item_values = dict() + item_values = self.recursive_dict_constructor() for name, values in self.items(): if item is None and not self.separator in name: @@ -111,8 +98,8 @@ def __getitem__(self, item: str = None): if self.separator in name: name_split = name.split(self.separator) - if str(name_split[-2]) == item and len(name_split) == 2: - item_values[name_split[-1]] = values + if str(name_split[0]) == item: + item_values[self.separator.join(name_split[1:])] = values return item_values diff --git a/testing/test_recursive_arguments.py b/testing/test_recursive_arguments.py index 20bccf67..857bd5e6 100644 --- a/testing/test_recursive_arguments.py +++ b/testing/test_recursive_arguments.py @@ -20,19 +20,21 @@ def test_recursive_arguments_should_get_root_level(): })} -def test_recursive_arguments_should_get_one_recursive_level(): +def test_recursive_arguments_should_get_recursive_levels(): ra = _RecursiveArguments(hyperparams=HyperparameterSamples({ 'hp0': 0, 'hp1': 1, 'stepa__hp2': 2, - 'stepb__hp3': 3 + 'stepb__hp3': 3, + 'stepb__stepd__hp4': 4 })) - ra = ra['stepa'] + ra = ra['stepb'] ra.kargs == [] ra.kwargs == {'hyperparams': HyperparameterSamples({ - 'stepa__hp2': 2 + 'stepb__hp3': 2, + 'stepb__stepd__hp4': 4 })} @@ -64,7 +66,7 @@ def test_has_children_mixin_apply_should_apply_method_to_direct_childrends(): p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ 'a__hp': 0, 'b__hp': 1, - 'Pipeline__hp': 2, + 'Pipeline__hp': 2 })) assert p['a'].get_hyperparams()['hp'] == 0 From eef5e86a8e4af789513d182ba2c1d88b1d9721d5 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 10 May 2020 15:41:58 -0400 Subject: [PATCH 17/48] Refactor recursive arguments in apply methods with _RecursiveArguments --- neuraxle/base.py | 13 ++- neuraxle/hyperparams/space.py | 26 +++--- neuraxle/metaopt/auto_ml.py | 4 +- neuraxle/metaopt/deprecated.py | 4 +- testing/test_apply_to_children.py | 11 --- testing/test_recursive_arguments.py | 15 ++-- testing/test_recursive_dict.py | 123 ++++++++++++++++++++++++++++ 7 files changed, 155 insertions(+), 41 deletions(-) delete mode 100644 testing/test_apply_to_children.py create mode 100644 testing/test_recursive_dict.py diff --git a/neuraxle/base.py b/neuraxle/base.py index ed7ae737..6a27df29 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -722,7 +722,7 @@ def set_savers(self, savers: List[BaseSaver]) -> 'BaseStep': self.savers: List[BaseSaver] = savers return self - def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': + def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> 'BaseStep': """ Set the step hyperparameters. @@ -742,10 +742,10 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': """ return self._set_hyperparams(hyperparams) - def _set_hyperparams(self, hyperparams): + def _set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]): self.invalidate() hyperparams = HyperparameterSamples(hyperparams).to_flat() - self.hyperparams = HyperparameterSamples(hyperparams) if len(hyperparams) > 0 else self.hyperparams + self.hyperparams = hyperparams if len(hyperparams) > 0 else self.hyperparams return self def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': @@ -1643,8 +1643,7 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a :return: """ ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *args, **kwargs) - results: RecursiveDict = RecursiveDict() - results[self.get_name()] = BaseStep.apply(self, method=method, ra=ra[None]) + results = BaseStep.apply(self, method=method, ra=ra[None]) results: RecursiveDict = self._apply_childrends(results=results, method=method, ra=ra) return results @@ -1696,7 +1695,7 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :func:`~BaseStep.get_hyperparams_space`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - return HyperparameterSpace(**self.apply(method='_get_hyperparams_space', ra=_RecursiveArguments())) + return HyperparameterSpace(self.apply(method='_get_hyperparams_space', ra=_RecursiveArguments())) def get_hyperparams(self) -> HyperparameterSamples: """ @@ -1707,7 +1706,7 @@ def get_hyperparams(self) -> HyperparameterSamples: :func:`~BaseStep.get_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - return HyperparameterSamples(**self.apply(method='_get_hyperparams', ra=_RecursiveArguments())) + return HyperparameterSamples(self.apply(method='_get_hyperparams', ra=_RecursiveArguments())) def set_train(self, is_train: bool = True) -> BaseStep: """ diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index 176f6b09..17081e69 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -79,18 +79,20 @@ class RecursiveDict(OrderedDict): """ DEFAULT_SEPARATOR = '__' - def __init__(self, separator=None, recursive_dict_constructor=None, *args, **kwds): - super().__init__(*args, **kwds) + def __init__(self, separator=None, *args, **kwds): + if len(args) == 1 and isinstance(args[0], RecursiveDict) and len(kwds) == 0: + super().__init__(args[0].items()) + separator = args[0].separator + else: + super().__init__(*args, **kwds) + if separator is None: separator = self.DEFAULT_SEPARATOR - if recursive_dict_constructor is None: - separator = RecursiveDict self.separator = separator - self.recursive_dict_constructor = recursive_dict_constructor def __getitem__(self, item: str = None): - item_values = self.recursive_dict_constructor() + item_values = type(self)() for name, values in self.items(): if item is None and not self.separator in name: @@ -109,15 +111,15 @@ def to_flat(self) -> 'RecursiveDict': :return: an HyperparameterSamples like self, flattened. """ - return nested_dict_to_flat(self, dict_ctor=self.recursive_dict_constructor) + return nested_dict_to_flat(self, dict_ctor=type(self)) - def to_nested_dict(self) -> 'RecursiveDict': + def flat_to_nested_dict(self) -> 'RecursiveDict': """ Will create an equivalent nested dict HyperparameterSamples. :return: an HyperparameterSamples like self, as a nested dict. """ - return flat_to_nested_dict(self, dict_ctor=self.recursive_dict_constructor) + return flat_to_nested_dict(self, dict_ctor=type(self)) def to_flat_as_dict_primitive(self) -> dict: """ @@ -190,7 +192,7 @@ def flat_to_nested_dict(flat_dict, dict_ctor=dict): else: ret[k] = v for k, v in pre_ret.items(): - ret[k] = flat_dict.flat_to_nested_dict(v, dict_ctor) + ret[k] = flat_to_nested_dict(v, dict_ctor) return ret @@ -204,7 +206,7 @@ class HyperparameterSamples(RecursiveDict): """ def __init__(self, *args, **kwds): - super().__init__(RecursiveDict.DEFAULT_SEPARATOR, HyperparameterSamples, *args, **kwds) + super().__init__(RecursiveDict.DEFAULT_SEPARATOR, *args, **kwds) class HyperparameterSpace(RecursiveDict): @@ -218,7 +220,7 @@ class HyperparameterSpace(RecursiveDict): """ def __init__(self, *args, **kwds): - super().__init__(RecursiveDict.DEFAULT_SEPARATOR, HyperparameterSpace, *args, **kwds) + super().__init__(RecursiveDict.DEFAULT_SEPARATOR, *args, **kwds) def rvs(self) -> 'HyperparameterSamples': """ diff --git a/neuraxle/metaopt/auto_ml.py b/neuraxle/metaopt/auto_ml.py index c3f590ba..485a026a 100644 --- a/neuraxle/metaopt/auto_ml.py +++ b/neuraxle/metaopt/auto_ml.py @@ -220,7 +220,7 @@ def new_trial(self, auto_ml_container: 'AutoMLContainer') -> 'Trial': :return: trial """ hyperparams = self.hyperparameter_selection_strategy.find_next_best_hyperparams(auto_ml_container) - self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4))) + self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) return Trial(hyperparams=hyperparams, main_metric_name=auto_ml_container.main_scoring_metric_name) @@ -680,7 +680,7 @@ def _fit_data_container(self, data_container: DataContainer, context: ExecutionC best_hyperparams = self.hyperparams_repository.get_best_hyperparams() self.print_func( - 'best hyperparams:\n{}'.format(json.dumps(best_hyperparams.to_nested_dict(), sort_keys=True, indent=4))) + 'best hyperparams:\n{}'.format(json.dumps(best_hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) p: BaseStep = self._load_virgin_model(hyperparams=best_hyperparams) if self.refit_trial: diff --git a/neuraxle/metaopt/deprecated.py b/neuraxle/metaopt/deprecated.py index b60ac243..24ce3a5d 100644 --- a/neuraxle/metaopt/deprecated.py +++ b/neuraxle/metaopt/deprecated.py @@ -160,7 +160,7 @@ def __init__(self, print_new_trial=True, print_success_trial=True, print_excepti def create_new_trial(self, hyperparams: HyperparameterSamples): if self.print_new_trial: - self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4))) + self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials': return self.trials.filter(status) @@ -171,7 +171,7 @@ def save_score_for_success_trial(self, hyperparams: HyperparameterSamples, score if self.print_success_trial: self.print_func('score: {}'.format(score)) self.print_func( - 'hyperparams:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4))) + 'hyperparams:\n{}'.format(json.dumps(hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) def save_failure_for_trial(self, hyperparams: HyperparameterSamples, exception: Exception): if self.print_exception: diff --git a/testing/test_apply_to_children.py b/testing/test_apply_to_children.py deleted file mode 100644 index c2674ece..00000000 --- a/testing/test_apply_to_children.py +++ /dev/null @@ -1,11 +0,0 @@ -from neuraxle.base import Identity -from neuraxle.pipeline import Pipeline - - -def test_apply_to_children(): - p = Pipeline([ - Identity(), - Identity() - ]) - - p.apply() diff --git a/testing/test_recursive_arguments.py b/testing/test_recursive_arguments.py index 857bd5e6..c874d663 100644 --- a/testing/test_recursive_arguments.py +++ b/testing/test_recursive_arguments.py @@ -69,9 +69,9 @@ def test_has_children_mixin_apply_should_apply_method_to_direct_childrends(): 'Pipeline__hp': 2 })) - assert p['a'].get_hyperparams()['hp'] == 0 - assert p['b'].get_hyperparams()['hp'] == 1 - assert p['Pipeline'].get_hyperparams()['hp'] == 2 + assert p['a'].hyperparams.to_flat_as_dict_primitive()['hp'] == 0 + assert p['b'].hyperparams.to_flat_as_dict_primitive()['hp'] == 1 + assert p['Pipeline'].hyperparams.to_flat_as_dict_primitive()['hp'] == 2 def test_has_children_mixin_apply_should_apply_method_to_recursive_childrends(): @@ -89,8 +89,8 @@ def test_has_children_mixin_apply_should_apply_method_to_recursive_childrends(): 'Pipeline__d__hp': 4 })) - assert p['Pipeline']['c'].get_hyperparams()['hp'] == 3 - assert p['Pipeline']['d'].get_hyperparams()['hp'] == 4 + assert p['Pipeline']['c'].hyperparams.to_flat_as_dict_primitive()['hp'] == 3 + assert p['Pipeline']['d'].hyperparams.to_flat_as_dict_primitive()['hp'] == 4 def test_has_children_mixin_apply_should_return_recursive_dict_to_direct_childrends(): @@ -101,8 +101,8 @@ def test_has_children_mixin_apply_should_return_recursive_dict_to_direct_childre results = p.apply('get_hyperparams', ra=None) - assert results['a__hp'] == 0 - assert results['b__hp'] == 0 + assert results.to_flat_as_dict_primitive()['a__hp'] == 0 + assert results.to_flat_as_dict_primitive()['b__hp'] == 1 def test_has_children_mixin_apply_should_return_recursive_dict_to_recursive_childrends(): @@ -114,6 +114,7 @@ def test_has_children_mixin_apply_should_return_recursive_dict_to_recursive_chil ]) results = p.apply('get_hyperparams', ra=None) + results = results.to_flat_as_dict_primitive() assert results['Pipeline__hp'] == 2 assert results['Pipeline__c__hp'] == 3 diff --git a/testing/test_recursive_dict.py b/testing/test_recursive_dict.py new file mode 100644 index 00000000..4559973e --- /dev/null +++ b/testing/test_recursive_dict.py @@ -0,0 +1,123 @@ +from neuraxle.hyperparams.space import RecursiveDict, HyperparameterSamples + + +def test_recursive_dict_to_flat(): + dict_values = { + 'hp': 1, + 'stepa': { + 'hp': 2, + 'stepb': { + 'hp': 3 + } + } + } + r = RecursiveDict(separator='__', **dict_values) + + r = r.to_flat() + + expected_dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + assert r == RecursiveDict(separator='__', **expected_dict_values) + + +def test_recursive_dict_to_nested_dict(): + dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + r = HyperparameterSamples(**dict_values) + + r = r.flat_to_nested_dict() + + expected_dict_values = { + 'hp': 1, + 'stepa': { + 'hp': 2, + 'stepb': { + 'hp': 3 + } + } + } + assert r == HyperparameterSamples(**expected_dict_values) + + +def test_recursive_dict_get_item(): + dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + r = HyperparameterSamples(**dict_values) + + assert r[None].to_flat_as_dict_primitive() == {'hp': 1} + assert r['stepa'].to_flat_as_dict_primitive() == {'hp': 2, 'stepb__hp': 3} + + +def test_hyperparams_to_nested_dict(): + dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + r = HyperparameterSamples(**dict_values) + + r = r.flat_to_nested_dict() + + expected_dict_values = { + 'hp': 1, + 'stepa': { + 'hp': 2, + 'stepb': { + 'hp': 3 + } + } + } + assert r.to_nested_dict_as_dict_primitive() == expected_dict_values + + +def test_recursive_dict_copy_constructor(): + dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + r = RecursiveDict('__', RecursiveDict(**dict_values)) + + assert r == RecursiveDict(**dict_values) + + +def test_hyperparams_copy_constructor(): + dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + r = HyperparameterSamples(HyperparameterSamples(**dict_values)) + + assert r == HyperparameterSamples(**dict_values) + + +def test_hyperparams_to_flat(): + dict_values = { + 'hp': 1, + 'stepa': { + 'hp': 2, + 'stepb': { + 'hp': 3 + } + } + } + r = HyperparameterSamples(**dict_values) + + r = r.to_flat() + + expected_dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + assert r == HyperparameterSamples(**expected_dict_values) From f2b4c3ea00decec9bc7b93a881b0311a04f0e532 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 10 May 2020 19:53:56 -0400 Subject: [PATCH 18/48] Fix __get_item__ RecursiveDict --- neuraxle/hyperparams/space.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index 17081e69..1594fe2f 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -103,8 +103,15 @@ def __getitem__(self, item: str = None): if str(name_split[0]) == item: item_values[self.separator.join(name_split[1:])] = values + if item == name: + item_values = values + return item_values + def __setitem__(self, key, value): + super().__setitem__(key, value) + pass + def to_flat(self) -> 'RecursiveDict': """ Will create an equivalent flat HyperparameterSamples. From e6d791a9fc4dff314a07cc7941ec537d7f4aff0b Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 10 May 2020 20:02:34 -0400 Subject: [PATCH 19/48] In _HasChildrenMix, only aggregate results when the output is a recursive dict --- neuraxle/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 6a27df29..ad71142f 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1650,7 +1650,11 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a def _apply_childrends(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): for children in self.get_children(): - results[children.get_name()] = children.apply(method=method, ra=ra[children.get_name()]) + children_results = children.apply(method=method, ra=ra[children.get_name()]) + + # aggregate results when the output type is a recursive dict. + if isinstance(children_results, RecursiveDict): + results[children.get_name()] = children_results return results @abstractmethod From 24ac7ddb8ae8b238eb758b57a41ccfa5e51ff0b2 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 10 May 2020 20:25:50 -0400 Subject: [PATCH 20/48] Fix Step Cloner get_children --- neuraxle/base.py | 4 +- neuraxle/steps/loop.py | 2 +- testing/test_apply.py | 149 ++++++++++++++++------------ testing/test_recursive_arguments.py | 69 +------------ 4 files changed, 91 insertions(+), 133 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index ad71142f..54a41313 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1317,8 +1317,8 @@ def _invalidate(step): if full_dump: # initialize and invalidate steps to make sure that all steps will be saved - self.apply_method(method=_initialize_if_needed) - self.apply_method(method=_invalidate) + self.apply(method=_initialize_if_needed) + self.apply(method=_invalidate) context.mkdir() stripped_step = copy(self) diff --git a/neuraxle/steps/loop.py b/neuraxle/steps/loop.py index 5c9927bb..c94105a5 100644 --- a/neuraxle/steps/loop.py +++ b/neuraxle/steps/loop.py @@ -164,7 +164,7 @@ def get_children(self) -> List[BaseStep]: :return: list of children """ children: List[BaseStep] = MetaStepMixin.get_children(self) - cloned_children = self.steps_as_tuple + cloned_children = [step for _, step in self.steps_as_tuple] children.extend(cloned_children) return children diff --git a/testing/test_apply.py b/testing/test_apply.py index 99c29a29..fb187a76 100644 --- a/testing/test_apply.py +++ b/testing/test_apply.py @@ -1,42 +1,28 @@ -import numpy as np - +from neuraxle.base import Identity from neuraxle.hyperparams.space import HyperparameterSamples from neuraxle.pipeline import Pipeline -from neuraxle.steps.misc import TapeCallbackFunction from neuraxle.steps.numpy import MultiplyByN from neuraxle.steps.output_handlers import OutputTransformerWrapper -DATA_INPUTS = np.array([1]) -EXPECTED_OUTPUTS = np.array([1]) - -tape_transform = TapeCallbackFunction() -tape_fit = TapeCallbackFunction() - - -def test_apply_on_pipeline_should_call_method_on_each_steps(): - pipeline = Pipeline([MultiplyByN(1), MultiplyByN(1)]) - - pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({'multiply_by': 2})) - - assert pipeline.get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN1'].get_hyperparams()['multiply_by'] == 2 - def test_apply_on_pipeline_with_positional_argument_should_call_method_on_each_steps(): pipeline = Pipeline([MultiplyByN(1), MultiplyByN(1)]) - pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({'multiply_by': 2})) + pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({ + 'multiply_by': 2, + 'MultiplyByN__multiply_by': 3, + 'MultiplyByN1__multiply_by': 4 + })) assert pipeline.get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN1'].get_hyperparams()['multiply_by'] == 2 + assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 3 + assert pipeline['MultiplyByN1'].get_hyperparams()['multiply_by'] == 4 def test_apply_method_on_pipeline_should_call_method_on_each_steps(): pipeline = Pipeline([MultiplyByN(1), MultiplyByN(1)]) - pipeline.apply_method( + pipeline.apply( lambda step: step.set_hyperparams(HyperparameterSamples({'multiply_by': 2})) ) @@ -45,59 +31,98 @@ def test_apply_method_on_pipeline_should_call_method_on_each_steps(): assert pipeline['MultiplyByN1'].get_hyperparams()['multiply_by'] == 2 -def test_apply_method_on_pipeline_with_positional_argument_should_call_method_on_each_steps(): - pipeline = Pipeline([MultiplyByN(1), MultiplyByN(1)]) - - pipeline.apply_method( - lambda step, hyperparams: step.set_hyperparams(hyperparams), - hyperparams=HyperparameterSamples({'multiply_by': 2}) - ) - - assert pipeline.get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN1'].get_hyperparams()['multiply_by'] == 2 - - -def test_apply_on_pipeline_with_meta_step_should_call_method_on_each_steps(): +def test_apply_on_pipeline_with_meta_step_and_positional_argument(): pipeline = Pipeline([OutputTransformerWrapper(MultiplyByN(1)), MultiplyByN(1)]) - pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({'multiply_by': 2})) + pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({ + 'multiply_by': 2, + 'OutputTransformerWrapper__multiply_by': 3, + 'OutputTransformerWrapper__MultiplyByN__multiply_by': 4, + 'MultiplyByN__multiply_by': 5 + })) assert pipeline.get_hyperparams()['multiply_by'] == 2 - assert pipeline['OutputTransformerWrapper'].wrapped.get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 2 - - -def test_apply_on_pipeline_with_meta_step_and_positional_argument_should_call_method_on_each_steps(): - pipeline = Pipeline([OutputTransformerWrapper(MultiplyByN(1)), MultiplyByN(1)]) - - pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({'multiply_by': 2})) - - assert pipeline.get_hyperparams()['multiply_by'] == 2 - assert pipeline['OutputTransformerWrapper'].wrapped.get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 2 + assert pipeline['OutputTransformerWrapper'].get_hyperparams()['multiply_by'] == 3 + assert pipeline['OutputTransformerWrapper'].wrapped.get_hyperparams()['multiply_by'] == 4 + assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 5 def test_apply_method_on_pipeline_with_meta_step_should_call_method_on_each_steps(): pipeline = Pipeline([OutputTransformerWrapper(MultiplyByN(1)), MultiplyByN(1)]) - pipeline.apply_method( + pipeline.apply( lambda step: step.set_hyperparams(HyperparameterSamples({'multiply_by': 2})) ) assert pipeline.get_hyperparams()['multiply_by'] == 2 + assert pipeline['OutputTransformerWrapper'].get_hyperparams()['multiply_by'] == 2 assert pipeline['OutputTransformerWrapper'].wrapped.get_hyperparams()['multiply_by'] == 2 assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 2 -def test_apply_method_on_pipeline_with_meta_step_and_positional_argument_should_call_method_on_each_steps(): - pipeline = Pipeline([OutputTransformerWrapper(MultiplyByN(1)), MultiplyByN(1)]) - - pipeline.apply_method( - lambda step, hyperparams: step.set_hyperparams(hyperparams), - hyperparams=HyperparameterSamples({'multiply_by': 2}) - ) - - assert pipeline.get_hyperparams()['multiply_by'] == 2 - assert pipeline['OutputTransformerWrapper'].wrapped.get_hyperparams()['multiply_by'] == 2 - assert pipeline['MultiplyByN'].get_hyperparams()['multiply_by'] == 2 +def test_has_children_mixin_apply_should_apply_method_to_direct_childrends(): + p = Pipeline([ + ('a', Identity()), + ('b', Identity()), + Pipeline([ + ('c', Identity()), + ('d', Identity()) + ]), + ]) + + p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ + 'a__hp': 0, + 'b__hp': 1, + 'Pipeline__hp': 2 + })) + + assert p['a'].hyperparams.to_flat_as_dict_primitive()['hp'] == 0 + assert p['b'].hyperparams.to_flat_as_dict_primitive()['hp'] == 1 + assert p['Pipeline'].hyperparams.to_flat_as_dict_primitive()['hp'] == 2 + + +def test_has_children_mixin_apply_should_apply_method_to_recursive_childrends(): + p = Pipeline([ + ('a', Identity()), + ('b', Identity()), + Pipeline([ + ('c', Identity()), + ('d', Identity()) + ]), + ]) + + p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ + 'Pipeline__c__hp': 3, + 'Pipeline__d__hp': 4 + })) + + assert p['Pipeline']['c'].hyperparams.to_flat_as_dict_primitive()['hp'] == 3 + assert p['Pipeline']['d'].hyperparams.to_flat_as_dict_primitive()['hp'] == 4 + + +def test_has_children_mixin_apply_should_return_recursive_dict_to_direct_childrends(): + p = Pipeline([ + ('a', Identity().set_hyperparams(HyperparameterSamples({'hp': 0}))), + ('b', Identity().set_hyperparams(HyperparameterSamples({'hp': 1}))) + ]) + + results = p.apply('get_hyperparams', ra=None) + + assert results.to_flat_as_dict_primitive()['a__hp'] == 0 + assert results.to_flat_as_dict_primitive()['b__hp'] == 1 + + +def test_has_children_mixin_apply_should_return_recursive_dict_to_recursive_childrends(): + p = Pipeline([ + Pipeline([ + ('c', Identity().set_hyperparams(HyperparameterSamples({'hp': 3}))), + ('d', Identity().set_hyperparams(HyperparameterSamples({'hp': 4}))) + ]).set_hyperparams(HyperparameterSamples({'hp': 2})), + ]) + + results = p.apply('get_hyperparams', ra=None) + results = results.to_flat_as_dict_primitive() + + assert results['Pipeline__hp'] == 2 + assert results['Pipeline__c__hp'] == 3 + assert results['Pipeline__d__hp'] == 4 diff --git a/testing/test_recursive_arguments.py b/testing/test_recursive_arguments.py index c874d663..c3768235 100644 --- a/testing/test_recursive_arguments.py +++ b/testing/test_recursive_arguments.py @@ -1,6 +1,5 @@ -from neuraxle.base import _RecursiveArguments, Identity +from neuraxle.base import _RecursiveArguments from neuraxle.hyperparams.space import HyperparameterSamples -from neuraxle.pipeline import Pipeline def test_recursive_arguments_should_get_root_level(): @@ -53,69 +52,3 @@ def test_recursive_arguments_should_have_copy_constructor(): })} -def test_has_children_mixin_apply_should_apply_method_to_direct_childrends(): - p = Pipeline([ - ('a', Identity()), - ('b', Identity()), - Pipeline([ - ('c', Identity()), - ('d', Identity()) - ]), - ]) - - p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ - 'a__hp': 0, - 'b__hp': 1, - 'Pipeline__hp': 2 - })) - - assert p['a'].hyperparams.to_flat_as_dict_primitive()['hp'] == 0 - assert p['b'].hyperparams.to_flat_as_dict_primitive()['hp'] == 1 - assert p['Pipeline'].hyperparams.to_flat_as_dict_primitive()['hp'] == 2 - - -def test_has_children_mixin_apply_should_apply_method_to_recursive_childrends(): - p = Pipeline([ - ('a', Identity()), - ('b', Identity()), - Pipeline([ - ('c', Identity()), - ('d', Identity()) - ]), - ]) - - p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ - 'Pipeline__c__hp': 3, - 'Pipeline__d__hp': 4 - })) - - assert p['Pipeline']['c'].hyperparams.to_flat_as_dict_primitive()['hp'] == 3 - assert p['Pipeline']['d'].hyperparams.to_flat_as_dict_primitive()['hp'] == 4 - - -def test_has_children_mixin_apply_should_return_recursive_dict_to_direct_childrends(): - p = Pipeline([ - ('a', Identity().set_hyperparams(HyperparameterSamples({'hp': 0}))), - ('b', Identity().set_hyperparams(HyperparameterSamples({'hp': 1}))) - ]) - - results = p.apply('get_hyperparams', ra=None) - - assert results.to_flat_as_dict_primitive()['a__hp'] == 0 - assert results.to_flat_as_dict_primitive()['b__hp'] == 1 - - -def test_has_children_mixin_apply_should_return_recursive_dict_to_recursive_childrends(): - p = Pipeline([ - Pipeline([ - ('c', Identity().set_hyperparams(HyperparameterSamples({'hp': 3}))), - ('d', Identity().set_hyperparams(HyperparameterSamples({'hp': 4}))) - ]).set_hyperparams(HyperparameterSamples({'hp': 2})), - ]) - - results = p.apply('get_hyperparams', ra=None) - results = results.to_flat_as_dict_primitive() - - assert results['Pipeline__hp'] == 2 - assert results['Pipeline__c__hp'] == 3 - assert results['Pipeline__d__hp'] == 4 From 1fdaad496df481b604895f28b76c6fdf46d3a74a Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 10 May 2020 20:46:55 -0400 Subject: [PATCH 21/48] Purge Apply Results --- neuraxle/base.py | 27 +++++++++++++++++++++++++-- testing/test_union.py | 2 +- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 54a41313..29c76670 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1643,20 +1643,43 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a :return: """ ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *args, **kwargs) - results = BaseStep.apply(self, method=method, ra=ra[None]) + results = RecursiveDict() + + results = self._apply_root(method, ra, results) results: RecursiveDict = self._apply_childrends(results=results, method=method, ra=ra) return results + def _apply_root(self, method, ra, results): + root_results = BaseStep.apply(self, method=method, ra=ra[None]) + root_results = self._purge_apply_results(root_results) + + if root_results is not None: + results = root_results + + return results + def _apply_childrends(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): for children in self.get_children(): children_results = children.apply(method=method, ra=ra[children.get_name()]) + children_results = self._purge_apply_results(children_results) + # aggregate results when the output type is a recursive dict. - if isinstance(children_results, RecursiveDict): + if children_results is not None: results[children.get_name()] = children_results + return results + def _purge_apply_results(self, results): + if isinstance(results, RecursiveDict): + return results + + if isinstance(results, dict) and not isinstance(results, RecursiveDict): + return RecursiveDict(**results) + + return None + @abstractmethod def get_children(self) -> List[BaseStep]: """ diff --git a/testing/test_union.py b/testing/test_union.py index 65583f1d..2f173796 100644 --- a/testing/test_union.py +++ b/testing/test_union.py @@ -78,7 +78,7 @@ def test_feature_union_should_apply_to_self_and_sub_steps(): ], joiner=NumpyTranspose()) ]) - p.apply('set_hyperparams', hyperparams=HyperparameterSamples({'applied': True})) + p.apply(lambda step: step.set_hyperparams(HyperparameterSamples({'applied': True}))) assert p.hyperparams['applied'] assert p['FeatureUnion'].hyperparams['applied'] From c7cdee5bfc9a9b812331211c74bb95606add5c60 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 11 May 2020 19:51:24 -0400 Subject: [PATCH 22/48] Fix All Unit Tests Related To Recursive Methods (but stuck with to_sklearn) --- examples/hyperparams/plot_hyperparams.py | 2 +- neuraxle/base.py | 87 +++++++++++++++++++----- neuraxle/hyperparams/space.py | 6 +- neuraxle/metaopt/auto_ml.py | 4 +- neuraxle/metaopt/deprecated.py | 4 +- neuraxle/steps/loop.py | 3 +- testing/test_deep_learning_pipeline.py | 8 +-- testing/test_pipeline.py | 14 ++-- testing/test_recursive_dict.py | 4 +- 9 files changed, 89 insertions(+), 43 deletions(-) diff --git a/examples/hyperparams/plot_hyperparams.py b/examples/hyperparams/plot_hyperparams.py index bdb58bcf..6fc34759 100644 --- a/examples/hyperparams/plot_hyperparams.py +++ b/examples/hyperparams/plot_hyperparams.py @@ -54,7 +54,7 @@ def main(): samples = p.get_hyperparams_space().rvs() p.set_hyperparams(samples) - samples = p.get_hyperparams() + samples = p.get_hyperparams().to_flat_as_dict_primitive() assert 42 <= samples['step1__multiply_by'] <= 50 assert -10 <= samples['step2__multiply_by'] <= 0 assert samples['Pipeline__PCA__n_components'] in [2, 3] diff --git a/neuraxle/base.py b/neuraxle/base.py index 29c76670..78b5123f 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -814,6 +814,9 @@ def set_params(self, **params) -> 'BaseStep': .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ + return self._set_params(params) + + def _set_params(self, params): return self.set_hyperparams(HyperparameterSamples(params)) def get_params(self) -> dict: @@ -833,6 +836,9 @@ def get_params(self) -> dict: .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples` """ + return self._get_params() + + def _get_params(self): return self.get_hyperparams().to_flat_as_ordered_dict_primitive() def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': @@ -854,10 +860,13 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ + return self._set_hyperparams_space(hyperparams_space) + + def _set_hyperparams_space(self, hyperparams_space): self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() - self.hyperparams_space = HyperparameterSpace(hyperparams_space) if len(hyperparams_space) > 0 else self.hyperparams_space - + self.hyperparams_space = HyperparameterSpace(hyperparams_space) if len( + hyperparams_space) > 0 else self.hyperparams_space return self def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': @@ -888,10 +897,12 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ + return self._update_hyperparams_space(hyperparams_space) + + def _update_hyperparams_space(self, hyperparams_space): self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() self.hyperparams_space.update(HyperparameterSpace(hyperparams_space).to_flat()) - return self def get_hyperparams_space(self) -> HyperparameterSpace: @@ -911,6 +922,9 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ + return self._get_hyperparams_space() + + def _get_hyperparams_space(self): return HyperparameterSpace(self.hyperparams_space.to_flat_as_dict_primitive()) def handle_inverse_transform(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: @@ -1612,17 +1626,6 @@ def __getitem__(self, item: str): def __iter__(self): return self.kwargs -# def apply(func, ra=None, *, **): -# ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *, **) -# result: RecursiveDict = BaseStep.apply(self, func, ra[None]) # ra[None] gives the root level and purges the childs in the returned new copy of RA. -# result: RecursiveDict = self.apply_childrends(self, result, func, ra) -# return result -# -# def apply_childrends(self, result: RecursiveDict, func: Union, ra: _RecursiveArguments) -> RecursiveDict: -# for children in self.get_kids(): -# result[children.get_name()] = children.apply(func, ra[children.get_name()]) -# return result - class _HasChildrenMixin: """ @@ -1698,7 +1701,9 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - self.apply(method='_update_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams)) + if not isinstance(hyperparams, RecursiveDict): + hyperparams = HyperparameterSamples(hyperparams) + self.apply(method='_update_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) return self def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': @@ -1710,7 +1715,9 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B :func:`~BaseStep.update_hyperparams_space`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - self.apply(method='_update_hyperparams_space', ra=_RecursiveArguments(hyperparams_space=hyperparams_space)) + if not isinstance(hyperparams_space, RecursiveDict): + hyperparams_space = HyperparameterSpace(hyperparams_space) + self.apply(method='_update_hyperparams_space', ra=_RecursiveArguments(hyperparams_space=hyperparams_space.to_flat())) return self def get_hyperparams_space(self) -> HyperparameterSpace: @@ -1722,7 +1729,8 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :func:`~BaseStep.get_hyperparams_space`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - return HyperparameterSpace(self.apply(method='_get_hyperparams_space', ra=_RecursiveArguments())) + results: HyperparameterSpace = self.apply(method='_get_hyperparams_space', ra=_RecursiveArguments()) + return results.to_flat() def get_hyperparams(self) -> HyperparameterSamples: """ @@ -1733,7 +1741,50 @@ def get_hyperparams(self) -> HyperparameterSamples: :func:`~BaseStep.get_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - return HyperparameterSamples(self.apply(method='_get_hyperparams', ra=_RecursiveArguments())) + results: HyperparameterSamples = self.apply(method='_get_hyperparams', ra=_RecursiveArguments()) + return results.to_flat() + + def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: + """ + Set all the pipeline hyperparams by applying :func:`~BaseStep.set_hyperparams` to all steps. + + .. seealso:: + :func:`~BaseStep.apply`, + :func:`~BaseStep.set_hyperparams`, + :class:`~neuraxle.hyperparams.space.HyperparameterSamples` + """ + if not isinstance(hyperparams, RecursiveDict): + hyperparams = HyperparameterSamples(hyperparams) + self.apply(method='_set_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) + return self + + def set_params(self, hyperparams: HyperparameterSamples) -> BaseStep: + """ + Set all the pipeline hyperparams by applying :func:`~BaseStep.set_hyperparams` to all steps. + + .. seealso:: + :func:`~BaseStep.apply`, + :func:`~BaseStep.set_hyperparams`, + :class:`~neuraxle.hyperparams.space.HyperparameterSamples` + """ + if not isinstance(hyperparams, RecursiveDict): + hyperparams = HyperparameterSamples(hyperparams) + self.apply(method='_set_params', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) + return self + + def set_hyperparams_space(self, hyperparams_space: HyperparameterSamples) -> BaseStep: + """ + Set all the pipeline hyperparams space by applying :func:`~BaseStep.set_hyperparams_space` to all steps. + + .. seealso:: + :func:`~BaseStep.apply`, + :func:`~BaseStep.set_hyperparams`, + :class:`~neuraxle.hyperparams.space.HyperparameterSpace` + """ + if not isinstance(hyperparams_space, RecursiveDict): + hyperparams_space = HyperparameterSpace(hyperparams_space) + self.apply(method='_set_hyperparams_space', ra=_RecursiveArguments(hyperparams_space=hyperparams_space.to_flat())) + return self def set_train(self, is_train: bool = True) -> BaseStep: """ diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index 1594fe2f..162cb6ed 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -108,10 +108,6 @@ def __getitem__(self, item: str = None): return item_values - def __setitem__(self, key, value): - super().__setitem__(key, value) - pass - def to_flat(self) -> 'RecursiveDict': """ Will create an equivalent flat HyperparameterSamples. @@ -120,7 +116,7 @@ def to_flat(self) -> 'RecursiveDict': """ return nested_dict_to_flat(self, dict_ctor=type(self)) - def flat_to_nested_dict(self) -> 'RecursiveDict': + def to_nested_dict(self) -> 'RecursiveDict': """ Will create an equivalent nested dict HyperparameterSamples. diff --git a/neuraxle/metaopt/auto_ml.py b/neuraxle/metaopt/auto_ml.py index 485a026a..c3f590ba 100644 --- a/neuraxle/metaopt/auto_ml.py +++ b/neuraxle/metaopt/auto_ml.py @@ -220,7 +220,7 @@ def new_trial(self, auto_ml_container: 'AutoMLContainer') -> 'Trial': :return: trial """ hyperparams = self.hyperparameter_selection_strategy.find_next_best_hyperparams(auto_ml_container) - self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) + self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4))) return Trial(hyperparams=hyperparams, main_metric_name=auto_ml_container.main_scoring_metric_name) @@ -680,7 +680,7 @@ def _fit_data_container(self, data_container: DataContainer, context: ExecutionC best_hyperparams = self.hyperparams_repository.get_best_hyperparams() self.print_func( - 'best hyperparams:\n{}'.format(json.dumps(best_hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) + 'best hyperparams:\n{}'.format(json.dumps(best_hyperparams.to_nested_dict(), sort_keys=True, indent=4))) p: BaseStep = self._load_virgin_model(hyperparams=best_hyperparams) if self.refit_trial: diff --git a/neuraxle/metaopt/deprecated.py b/neuraxle/metaopt/deprecated.py index 24ce3a5d..b60ac243 100644 --- a/neuraxle/metaopt/deprecated.py +++ b/neuraxle/metaopt/deprecated.py @@ -160,7 +160,7 @@ def __init__(self, print_new_trial=True, print_success_trial=True, print_excepti def create_new_trial(self, hyperparams: HyperparameterSamples): if self.print_new_trial: - self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) + self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4))) def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials': return self.trials.filter(status) @@ -171,7 +171,7 @@ def save_score_for_success_trial(self, hyperparams: HyperparameterSamples, score if self.print_success_trial: self.print_func('score: {}'.format(score)) self.print_func( - 'hyperparams:\n{}'.format(json.dumps(hyperparams.flat_to_nested_dict(), sort_keys=True, indent=4))) + 'hyperparams:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4))) def save_failure_for_trial(self, hyperparams: HyperparameterSamples, exception: Exception): if self.print_exception: diff --git a/neuraxle/steps/loop.py b/neuraxle/steps/loop.py index c94105a5..83c8e8a4 100644 --- a/neuraxle/steps/loop.py +++ b/neuraxle/steps/loop.py @@ -182,8 +182,7 @@ def _copy_one_step_per_data_input(self, data_container): self.steps_as_tuple = [(step.name, step) for step in steps] self.invalidate() - def _fit_transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> ( - 'BaseStep', DataContainer): + def _fit_transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> ('BaseStep', DataContainer): fitted_steps_data_containers = [] for i, (current_ids, data_inputs, expected_outputs) in enumerate(data_container): fitted_step_data_container = self[i].handle_fit_transform( diff --git a/testing/test_deep_learning_pipeline.py b/testing/test_deep_learning_pipeline.py index 1a60bd17..0a86941b 100644 --- a/testing/test_deep_learning_pipeline.py +++ b/testing/test_deep_learning_pipeline.py @@ -37,13 +37,13 @@ def test_deep_learning_pipeline(): # When p, outputs = p.fit_transform(data_inputs, expected_outputs) - metrics = p.apply('get_metrics') + metrics = p.apply('get_metrics').to_flat_as_dict_primitive() # Then - batch_mse_train = metrics['DeepLearningPipeline__EpochRepeater__validation_split_wrapper__epoch_metrics']['train']['mse'] - epoch_mse_train = metrics['DeepLearningPipeline__EpochRepeater__validation_split_wrapper__epoch_metrics__TrainShuffled__MiniBatchSequentialPipeline__batch_metrics']['train']['mse'] + batch_mse_train = metrics['EpochRepeater__validation_split_wrapper__epoch_metrics__train__mse'] + epoch_mse_train = metrics['EpochRepeater__validation_split_wrapper__epoch_metrics__TrainShuffled__MiniBatchSequentialPipeline__batch_metrics__train__mse'] - epoch_mse_validation = metrics['DeepLearningPipeline__EpochRepeater__validation_split_wrapper__epoch_metrics']['validation']['mse'] + epoch_mse_validation = metrics['EpochRepeater__validation_split_wrapper__epoch_metrics__validation__mse'] assert len(epoch_mse_train) == N_EPOCHS assert len(epoch_mse_validation) == N_EPOCHS diff --git a/testing/test_pipeline.py b/testing/test_pipeline.py index d31a49dc..7c66d119 100644 --- a/testing/test_pipeline.py +++ b/testing/test_pipeline.py @@ -117,9 +117,9 @@ def test_pipeline_set_one_hyperparam_level_one_flat(): "a__learning_rate": 7 }) - assert p["a"].hyperparams["learning_rate"] == 7 - assert p["b"].hyperparams == dict() - assert p["c"].hyperparams == dict() + assert p["a"].hyperparams.to_flat_as_dict_primitive()["learning_rate"] == 7 + assert p["b"].hyperparams.to_flat_as_dict_primitive() == dict() + assert p["c"].hyperparams.to_flat_as_dict_primitive() == dict() def test_pipeline_set_one_hyperparam_level_one_dict(): @@ -183,10 +183,10 @@ def test_pipeline_set_one_hyperparam_level_two_dict(): }) print(p.get_hyperparams()) - assert p["b"]["a"].hyperparams["learning_rate"] == 7 - assert p["b"]["c"].hyperparams == dict() - assert p["b"].hyperparams["learning_rate"] == 9 - assert p["c"].hyperparams == dict() + assert p["b"]["a"].get_hyperparams()["learning_rate"] == 7 + assert p["b"]["c"].get_hyperparams() == dict() + assert p["b"].get_hyperparams()["learning_rate"] == 9 + assert p["c"].get_hyperparams() == dict() def test_pipeline_update_hyperparam_level_one_flat(): diff --git a/testing/test_recursive_dict.py b/testing/test_recursive_dict.py index 4559973e..9de72f12 100644 --- a/testing/test_recursive_dict.py +++ b/testing/test_recursive_dict.py @@ -31,7 +31,7 @@ def test_recursive_dict_to_nested_dict(): } r = HyperparameterSamples(**dict_values) - r = r.flat_to_nested_dict() + r = r.to_nested_dict() expected_dict_values = { 'hp': 1, @@ -65,7 +65,7 @@ def test_hyperparams_to_nested_dict(): } r = HyperparameterSamples(**dict_values) - r = r.flat_to_nested_dict() + r = r.to_nested_dict() expected_dict_values = { 'hp': 1, From 06179cbef9bf3b9821cf239166197a2d4ed5e426 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 11 May 2020 20:21:14 -0400 Subject: [PATCH 23/48] Fix Broken Set hyperparams space for sklearn --- neuraxle/steps/sklearn.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index 93dede3a..450a3eae 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -24,12 +24,12 @@ """ import inspect -from typing import Any +from typing import Any, List from sklearn.base import BaseEstimator from sklearn.linear_model import Ridge -from neuraxle.base import BaseStep +from neuraxle.base import BaseStep, _HasChildrenMixin from neuraxle.hyperparams.distributions import LogUniform, Boolean from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples from neuraxle.steps.numpy import NumpyTranspose @@ -86,18 +86,17 @@ def _set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: :return: self """ # flatten the step hyperparams, and set the wrapped sklearn predictor params - self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(hyperparams).to_flat_as_dict_primitive()) - - # there is remainders when we need to set params for steps that are inside sklearn.pipeline.Pipeline... - self.wrapped_sklearn_predictor.set_params(**HyperparameterSamples(HyperparameterSamples(hyperparams).to_flat()).to_flat_as_dict_primitive()) + hyperparams = HyperparameterSamples(hyperparams) + BaseStep._set_hyperparams(self, hyperparams.to_flat()) + self.wrapped_sklearn_predictor.set_params(**hyperparams.to_flat_as_dict_primitive()) return self - def get_hyperparams(self): + def _get_hyperparams(self): if self.return_all_sklearn_default_params_on_get: return HyperparameterSamples(self.wrapped_sklearn_predictor.get_params()).to_flat() else: - return BaseStep.get_hyperparams(self) + return BaseStep._get_hyperparams(self) def get_wrapped_sklearn_predictor(self): return self.wrapped_sklearn_predictor From 2ad630bc5c273e6bf8f3872c0d70a7f3bf6e3896 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Tue, 12 May 2020 22:57:33 -0400 Subject: [PATCH 24/48] Add recursive methods docstring --- neuraxle/base.py | 63 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 78b5123f..5c51867d 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -638,9 +638,14 @@ def setup(self) -> 'BaseStep': def invalidate(self) -> 'BaseStep': """ - Invalidate step. + Invalidate a step, and all of its children. :return: self + .. note:: + This is a recursive method used in :class:̀_HasChildrenMixin`. + .. seealso:: + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply` """ return self._invalidate() @@ -666,8 +671,12 @@ def set_train(self, is_train: bool = True): :param is_train: is training mode or not :return: + .. note:: + This is a recursive method used in :class:̀_HasChildrenMixin`. .. seealso:: - :func:`BaseStep.set_train` + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply`, + :func:`_HasChildrenMixin._set_train` """ return self._set_train(is_train) @@ -737,8 +746,14 @@ def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> 'B :param hyperparams: hyperparameters :return: self + .. note:: + This is a recursive method. .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` + :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, + :class:̀_HasChildrenMixin`, + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply`, + :func:`_HasChildrenMixin._set_train` """ return self._set_hyperparams(hyperparams) @@ -772,9 +787,14 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': :param hyperparams: hyperparameters :return: self + .. note:: + This is a recursive method. .. seealso:: - :func:`~BaseStep.update_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` + :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, + :class:̀_HasChildrenMixin`, + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply`, + :func:`_HasChildrenMixin._update_hyperparams` """ self._update_hyperparams(hyperparams) @@ -789,8 +809,14 @@ def get_hyperparams(self) -> HyperparameterSamples: :return: step hyperparameters + .. note:: + This is a recursive method. .. seealso:: - * :class:`~neuraxle.hyperparams.space.HyperparameterSamples` + :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, + :class:̀_HasChildrenMixin`, + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply`, + :func:`_HasChildrenMixin._get_hyperparams` """ return self._get_hyperparams() @@ -811,8 +837,14 @@ def set_params(self, **params) -> 'BaseStep': :param **params: arbitrary number of arguments for hyperparameters + .. note:: + This is a recursive method. .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` + :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, + :class:̀_HasChildrenMixin`, + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply`, + :func:`_HasChildrenMixin._set_params` """ return self._set_params(params) @@ -833,8 +865,14 @@ def get_params(self) -> dict: :return: hyperparameters + .. note:: + This is a recursive method. .. seealso:: - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` + :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, + :class:̀_HasChildrenMixin`, + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply`, + :func:`_HasChildrenMixin._get_params` """ return self._get_params() @@ -859,6 +897,15 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` + + .. note:: + This is a recursive method. + .. seealso:: + :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, + :class:̀_HasChildrenMixin`, + :func:`BaseStep.apply`, + :func:`_HasChildrenMixin._apply`, + :func:`_HasChildrenMixin._get_params` """ return self._set_hyperparams_space(hyperparams_space) From c49603a831bb09fb3de4912ae31ff7520ccf0ed6 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 12:13:01 -0400 Subject: [PATCH 25/48] Add Docstring For Invalidate --- neuraxle/base.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 5c51867d..b239455d 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -638,7 +638,15 @@ def setup(self) -> 'BaseStep': def invalidate(self) -> 'BaseStep': """ - Invalidate a step, and all of its children. + Invalidate a step, and all of its children. Invalidating a step makes it eligible to be saved again. + + A step is invalidated when any of the following things happen : + * a mutation has been performed on the step : func:`~.mutate` + * an hyperparameter has changed func:`~.set_hyperparams` + * an hyperparameter space has changed func:`~.set_hyperparams_space` + * a call to the fit method func:`~.handle_fit` + * a call to the fit_transform method func:`~.handle_fit_transform` + * the step name has changed func:`~neuraxle.base.BaseStep.set_name` :return: self .. note:: From 9b58bd7f28ddb20d0f9cfa65e16adf5c21b25b14 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 12:47:58 -0400 Subject: [PATCH 26/48] Remove Private _apply method from BaseStep --- neuraxle/base.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index b239455d..ced01c0e 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1015,21 +1015,15 @@ def apply(self, method: Union[str, Callable], ra: '_RecursiveArguments' = None, :param ra: recursive arguments :param args: any additional arguments to be passed to the method :param kwargs: any additional positional arguments to be passed to the method - :return: accumulated results + :return: method outputs, or None if no method has been applied + + .. seealso:: + :class:`_RecursiveArguments`, + :class:`_HasChildrenMixin` """ if ra is None: ra = _RecursiveArguments(*args, **kwargs) - return self._apply(method=method, ra=ra) - - def _apply(self, method: str, ra: '_RecursiveArguments') -> Any: - """ - Apply a method to a step and its children. - - :param method: method name that need to be called on all steps - :param ra: current pipeline step name - :return: method outputs, or None if no method has been applied - """ if isinstance(method, Callable): return method(self, *ra.kargs, **ra.kwargs) elif hasattr(self, method) and callable(getattr(self, method)): @@ -1633,7 +1627,6 @@ def _sklearn_to_neuraxle_step(step) -> BaseStep: step.set_name(step.get_wrapped_sklearn_predictor().__class__.__name__) return step - class _RecursiveArguments: def __init__(self, ra=None, *kargs, **kwargs): if ra is not None: From 26019facc2304aca5a5530b04a1e4fd697bdcf76 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 12:57:18 -0400 Subject: [PATCH 27/48] Remove None In apply results --- neuraxle/base.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index ced01c0e..3e3162d9 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1029,7 +1029,7 @@ def apply(self, method: Union[str, Callable], ra: '_RecursiveArguments' = None, elif hasattr(self, method) and callable(getattr(self, method)): return getattr(self, method)(*ra.kargs, **ra.kwargs) - return None + return RecursiveDict() def get_step_by_name(self, name): if self.name == name: @@ -1694,31 +1694,22 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a :return: """ ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *args, **kwargs) - results = RecursiveDict() - - results = self._apply_root(method, ra, results) + results = self._apply_root(method, ra) results: RecursiveDict = self._apply_childrends(results=results, method=method, ra=ra) return results - def _apply_root(self, method, ra, results): + def _apply_root(self, method: Union[str, Callable], ra: _RecursiveArguments): root_results = BaseStep.apply(self, method=method, ra=ra[None]) root_results = self._purge_apply_results(root_results) - - if root_results is not None: - results = root_results - - return results + return root_results def _apply_childrends(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): for children in self.get_children(): children_results = children.apply(method=method, ra=ra[children.get_name()]) children_results = self._purge_apply_results(children_results) - - # aggregate results when the output type is a recursive dict. - if children_results is not None: - results[children.get_name()] = children_results + results[children.get_name()] = children_results return results @@ -1729,7 +1720,7 @@ def _purge_apply_results(self, results): if isinstance(results, dict) and not isinstance(results, RecursiveDict): return RecursiveDict(**results) - return None + return RecursiveDict() @abstractmethod def get_children(self) -> List[BaseStep]: From 131942e7586216b27cbbe45b53b8e52b37c3a8a3 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 13:16:01 -0400 Subject: [PATCH 28/48] Move _RecursiveArguments Definition on top of base.py for guillaume --- neuraxle/base.py | 110 ++++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 48 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 3e3162d9..cb1a7f6b 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -485,6 +485,67 @@ def __len__(self): return len(self.parents) +class _RecursiveArguments: + """ + This class is used by :func:`~neuraxle.base.BaseStep.apply`, and :class:`_HasChildrenMixin` to pass the right arguments to steps with children. + + .. seealso:: + :class:`_HasChildrenMixin`, + :func:`~neuraxle.base.BaseStep.set_hyperparams_space`, + :func:`~neuraxle.base.BaseStep.get_hyperparams_space`, + :func:`~neuraxle.base.BaseStep.get_hyperparams`, + :func:`~neuraxle.base.BaseStep.set_hyperparams`, + :func:`~neuraxle.base.BaseStep.update_hyperparams`, + :func:`~neuraxle.base.BaseStep.update_hyperparams_space`, + :func:`~neuraxle.base.BaseStep.invalidate` + """ + def __init__(self, ra=None, *kargs, **kwargs): + if ra is not None: + kargs = ra.kargs + kwargs = ra.kwargs + self.kargs = kargs + self.kwargs = kwargs + + def __getitem__(self, item: str): + """ + Return arguments for the given path. + + :param item: root name + :return: + """ + if item is None: + arguments = list() + keyword_arguments = dict() + for arg in self.kargs: + if isinstance(arg, RecursiveDict): + arguments.append(arg[item]) + else: + arguments.append(arg) + for key in self.kwargs: + if isinstance(self.kwargs[key], RecursiveDict): + keyword_arguments[key] = self.kwargs[key][item] + else: + keyword_arguments[key] = self.kwargs[key] + return _RecursiveArguments(*arguments, **keyword_arguments) + else: + arguments = list() + keyword_arguments = dict() + for arg in self.kargs: + if isinstance(arg, RecursiveDict): + arguments.append(arg[item]) + else: + arguments.append(arg) + for key in self.kwargs: + if isinstance(self.kwargs[key], RecursiveDict): + keyword_arguments[key] = self.kwargs[key][item] + else: + keyword_arguments[key] = self.kwargs[key] + return _RecursiveArguments(*arguments, **keyword_arguments) + + def __iter__(self): + return self.kwargs + + class BaseStep(ABC): """ Base class for a pipeline step. @@ -1007,7 +1068,7 @@ def _inverse_transform_data_container(self, data_container: DataContainer, conte return data_container - def apply(self, method: Union[str, Callable], ra: '_RecursiveArguments' = None, *args, **kwargs) -> RecursiveDict: + def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *args, **kwargs) -> RecursiveDict: """ Apply a method to a step and its children. @@ -1627,53 +1688,6 @@ def _sklearn_to_neuraxle_step(step) -> BaseStep: step.set_name(step.get_wrapped_sklearn_predictor().__class__.__name__) return step -class _RecursiveArguments: - def __init__(self, ra=None, *kargs, **kwargs): - if ra is not None: - kargs = ra.kargs - kwargs = ra.kwargs - self.kargs = kargs - self.kwargs = kwargs - - def __getitem__(self, item: str): - """ - Return arguments for the given path. - - :param item: root name - :return: - """ - if item is None: - arguments = list() - keyword_arguments = dict() - for arg in self.kargs: - if isinstance(arg, RecursiveDict): - arguments.append(arg[item]) - else: - arguments.append(arg) - for key in self.kwargs: - if isinstance(self.kwargs[key], RecursiveDict): - keyword_arguments[key] = self.kwargs[key][item] - else: - keyword_arguments[key] = self.kwargs[key] - return _RecursiveArguments(*arguments, **keyword_arguments) - else: - arguments = list() - keyword_arguments = dict() - for arg in self.kargs: - if isinstance(arg, RecursiveDict): - arguments.append(arg[item]) - else: - arguments.append(arg) - for key in self.kwargs: - if isinstance(self.kwargs[key], RecursiveDict): - keyword_arguments[key] = self.kwargs[key][item] - else: - keyword_arguments[key] = self.kwargs[key] - return _RecursiveArguments(*arguments, **keyword_arguments) - - def __iter__(self): - return self.kwargs - class _HasChildrenMixin: """ From 988af658993748c530dcfcd072972cc0549133b7 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 13:18:38 -0400 Subject: [PATCH 29/48] _RecursiveArguments: Use .items() in kwargs loops --- neuraxle/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index cb1a7f6b..056485f7 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -521,11 +521,11 @@ def __getitem__(self, item: str): arguments.append(arg[item]) else: arguments.append(arg) - for key in self.kwargs: - if isinstance(self.kwargs[key], RecursiveDict): - keyword_arguments[key] = self.kwargs[key][item] + for key, arg in self.kwargs.items(): + if isinstance(arg, RecursiveDict): + keyword_arguments[key] = arg[item] else: - keyword_arguments[key] = self.kwargs[key] + keyword_arguments[key] = arg return _RecursiveArguments(*arguments, **keyword_arguments) else: arguments = list() @@ -535,11 +535,11 @@ def __getitem__(self, item: str): arguments.append(arg[item]) else: arguments.append(arg) - for key in self.kwargs: - if isinstance(self.kwargs[key], RecursiveDict): - keyword_arguments[key] = self.kwargs[key][item] + for key, arg in self.kwargs.items(): + if isinstance(arg, RecursiveDict): + keyword_arguments[key] = arg[item] else: - keyword_arguments[key] = self.kwargs[key] + keyword_arguments[key] = arg return _RecursiveArguments(*arguments, **keyword_arguments) def __iter__(self): From f46777332d4cbd5b226b7919478285ff14056fc5 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 13:23:39 -0400 Subject: [PATCH 30/48] Fix _RecursiveArguments __getitem__ --- neuraxle/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 056485f7..2d36535f 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -506,24 +506,24 @@ def __init__(self, ra=None, *kargs, **kwargs): self.kargs = kargs self.kwargs = kwargs - def __getitem__(self, item: str): + def __getitem__(self, child_step_name: str): """ Return arguments for the given path. - :param item: root name - :return: + :param child_step_name: child step name, or None if we want to get root values. + :return: recursive argument for the given child step name """ - if item is None: + if child_step_name is None: arguments = list() keyword_arguments = dict() for arg in self.kargs: if isinstance(arg, RecursiveDict): - arguments.append(arg[item]) + arguments.append(arg[child_step_name]) else: arguments.append(arg) for key, arg in self.kwargs.items(): if isinstance(arg, RecursiveDict): - keyword_arguments[key] = arg[item] + keyword_arguments[key] = arg[child_step_name] else: keyword_arguments[key] = arg return _RecursiveArguments(*arguments, **keyword_arguments) @@ -532,12 +532,12 @@ def __getitem__(self, item: str): keyword_arguments = dict() for arg in self.kargs: if isinstance(arg, RecursiveDict): - arguments.append(arg[item]) + arguments.append(arg[child_step_name]) else: arguments.append(arg) for key, arg in self.kwargs.items(): if isinstance(arg, RecursiveDict): - keyword_arguments[key] = arg[item] + keyword_arguments[key] = arg[child_step_name] else: keyword_arguments[key] = arg return _RecursiveArguments(*arguments, **keyword_arguments) From 87cbfde81f3b8cc864a35bc3fd0888271dcabe15 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 13:27:36 -0400 Subject: [PATCH 31/48] Fix Typo --- neuraxle/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 2d36535f..3b9c2b75 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -508,7 +508,8 @@ def __init__(self, ra=None, *kargs, **kwargs): def __getitem__(self, child_step_name: str): """ - Return arguments for the given path. + Return recursive arguments for the given child step name. + If child step name is None, return the root values. :param child_step_name: child step name, or None if we want to get root values. :return: recursive argument for the given child step name @@ -1709,7 +1710,7 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a """ ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *args, **kwargs) results = self._apply_root(method, ra) - results: RecursiveDict = self._apply_childrends(results=results, method=method, ra=ra) + results: RecursiveDict = self._apply_childrens(results=results, method=method, ra=ra) return results @@ -1718,7 +1719,7 @@ def _apply_root(self, method: Union[str, Callable], ra: _RecursiveArguments): root_results = self._purge_apply_results(root_results) return root_results - def _apply_childrends(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): + def _apply_childrens(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): for children in self.get_children(): children_results = children.apply(method=method, ra=ra[children.get_name()]) From e33eb259544d2aca4ed61ef0f0f846d5a0c14e53 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 13:28:57 -0400 Subject: [PATCH 32/48] Fix Typings In _HasChildrenMixin --- neuraxle/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 3b9c2b75..69b6641c 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1709,17 +1709,17 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a :return: """ ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *args, **kwargs) - results = self._apply_root(method, ra) + results: RecursiveDict = self._apply_root(method, ra) results: RecursiveDict = self._apply_childrens(results=results, method=method, ra=ra) return results def _apply_root(self, method: Union[str, Callable], ra: _RecursiveArguments): - root_results = BaseStep.apply(self, method=method, ra=ra[None]) - root_results = self._purge_apply_results(root_results) + root_results: RecursiveDict = BaseStep.apply(self, method=method, ra=ra[None]) + root_results: RecursiveDict = self._purge_apply_results(root_results) return root_results - def _apply_childrens(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments): + def _apply_childrens(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments) -> RecursiveDict: for children in self.get_children(): children_results = children.apply(method=method, ra=ra[children.get_name()]) @@ -1728,7 +1728,7 @@ def _apply_childrens(self, results: RecursiveDict, method: Union[str, Callable], return results - def _purge_apply_results(self, results): + def _purge_apply_results(self, results) -> RecursiveDict: if isinstance(results, RecursiveDict): return results From b9339f001c76ba83ef75edbf8dcb28ff94161e80 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:07:52 -0400 Subject: [PATCH 33/48] Refactor else if in apply method, and add try catch --- neuraxle/base.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 69b6641c..df306823 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -28,6 +28,7 @@ import inspect import os import pprint +import traceback import warnings from abc import ABC, abstractmethod from collections import OrderedDict @@ -1086,12 +1087,18 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a if ra is None: ra = _RecursiveArguments(*args, **kwargs) - if isinstance(method, Callable): - return method(self, *ra.kargs, **ra.kwargs) - elif hasattr(self, method) and callable(getattr(self, method)): - return getattr(self, method)(*ra.kargs, **ra.kwargs) + kargs = ra.kargs - return RecursiveDict() + if isinstance(method, str) and hasattr(self, method) and callable(getattr(self, method)): + method = getattr(self, method) + else: + kargs = [self] + list(kargs) + + try: + return method(*kargs, **ra.kwargs) + except Exception: + print('{}: Failed to apply method {}.'.format(self.name, method)) + print(traceback.format_stack()) def get_step_by_name(self, name): if self.name == name: From cbbadc3aa393c72ceaa9a42a1220eca87cc20c1a Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:09:49 -0400 Subject: [PATCH 34/48] Extract variable for terminal ra --- neuraxle/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index df306823..47fdadac 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1722,7 +1722,8 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a return results def _apply_root(self, method: Union[str, Callable], ra: _RecursiveArguments): - root_results: RecursiveDict = BaseStep.apply(self, method=method, ra=ra[None]) + terminal_ra: _RecursiveArguments = ra[None] + root_results: RecursiveDict = BaseStep.apply(self, method=method, ra=terminal_ra) root_results: RecursiveDict = self._purge_apply_results(root_results) return root_results From bfad161c155ce23c049b4350f34653f3dda84b61 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:12:54 -0400 Subject: [PATCH 35/48] Rename Self Results --- neuraxle/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 47fdadac..7a7c798f 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1716,16 +1716,16 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a :return: """ ra: _RecursiveArguments = _RecursiveArguments(ra=ra, *args, **kwargs) - results: RecursiveDict = self._apply_root(method, ra) + results: RecursiveDict = self._apply_self(method=method, ra=ra) results: RecursiveDict = self._apply_childrens(results=results, method=method, ra=ra) return results - def _apply_root(self, method: Union[str, Callable], ra: _RecursiveArguments): + def _apply_self(self, method: Union[str, Callable], ra: _RecursiveArguments): terminal_ra: _RecursiveArguments = ra[None] - root_results: RecursiveDict = BaseStep.apply(self, method=method, ra=terminal_ra) - root_results: RecursiveDict = self._purge_apply_results(root_results) - return root_results + self_results: RecursiveDict = BaseStep.apply(self, method=method, ra=terminal_ra) + self_results: RecursiveDict = self._purge_apply_results(self_results) + return self_results def _apply_childrens(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments) -> RecursiveDict: for children in self.get_children(): From fbbc72ddaaa2cdcdade884a3563b699c115e9aa2 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:18:10 -0400 Subject: [PATCH 36/48] Fix set hyperparams space typings in _HasChildren Mixin --- neuraxle/base.py | 2 +- testing/hyperparams/test_get_set_hyperparams.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 7a7c798f..01806a6a 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1834,7 +1834,7 @@ def set_params(self, hyperparams: HyperparameterSamples) -> BaseStep: self.apply(method='_set_params', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) return self - def set_hyperparams_space(self, hyperparams_space: HyperparameterSamples) -> BaseStep: + def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> BaseStep: """ Set all the pipeline hyperparams space by applying :func:`~BaseStep.set_hyperparams_space` to all steps. diff --git a/testing/hyperparams/test_get_set_hyperparams.py b/testing/hyperparams/test_get_set_hyperparams.py index 36edb8f9..c6b6aa4b 100644 --- a/testing/hyperparams/test_get_set_hyperparams.py +++ b/testing/hyperparams/test_get_set_hyperparams.py @@ -89,7 +89,7 @@ def test_step_cloner_update_hyperparams_should_update_wrapped_step_hyperparams() })) p.update_hyperparams(HyperparameterSamples({ - SOME_STEP_HP: SOME_STEP_HP_VALUE + 1, + SOME_STEP_HP: SOME_STEP_HP_VALUE + 1 })) assert isinstance(p.hyperparams, HyperparameterSamples) From 8ee2193ece42fafefafd6780c3bb99b43d66e8ab Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:26:01 -0400 Subject: [PATCH 37/48] Pass dict in unit tests for get set hyperparams --- neuraxle/base.py | 24 ++-- .../hyperparams/test_get_set_hyperparams.py | 120 +++++++++--------- 2 files changed, 72 insertions(+), 72 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 01806a6a..e5380674 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -834,7 +834,7 @@ def _set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]): self.hyperparams = hyperparams if len(hyperparams) > 0 else self.hyperparams return self - def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': + def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> 'BaseStep': """ Update the step hyperparameters without removing the already-set hyperparameters. This can be useful to add more hyperparameters to the existing ones without flushing the ones that were already set. @@ -871,7 +871,7 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': return self - def _update_hyperparams(self, hyperparams: HyperparameterSamples): + def _update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]): self.hyperparams.update(HyperparameterSamples(hyperparams).to_flat()) def get_hyperparams(self) -> HyperparameterSamples: @@ -919,7 +919,7 @@ def set_params(self, **params) -> 'BaseStep': """ return self._set_params(params) - def _set_params(self, params): + def _set_params(self, params: dict): return self.set_hyperparams(HyperparameterSamples(params)) def get_params(self) -> dict: @@ -950,7 +950,7 @@ def get_params(self) -> dict: def _get_params(self): return self.get_hyperparams().to_flat_as_ordered_dict_primitive() - def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': + def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> 'BaseStep': """ Set step hyperparameters space. @@ -980,14 +980,14 @@ def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'Base """ return self._set_hyperparams_space(hyperparams_space) - def _set_hyperparams_space(self, hyperparams_space): + def _set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() self.hyperparams_space = HyperparameterSpace(hyperparams_space) if len( hyperparams_space) > 0 else self.hyperparams_space return self - def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': + def update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> 'BaseStep': """ Update the step hyperparameter spaces without removing the already-set hyperparameters. This can be useful to add more hyperparameter spaces to the existing ones without flushing the ones that were already set. @@ -1017,7 +1017,7 @@ def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'B """ return self._update_hyperparams_space(hyperparams_space) - def _update_hyperparams_space(self, hyperparams_space): + def _update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): self.invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() self.hyperparams_space.update(HyperparameterSpace(hyperparams_space).to_flat()) @@ -1754,7 +1754,7 @@ def get_children(self) -> List[BaseStep]: """ pass - def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': + def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> 'BaseStep': """ Update hyperparams recursively by applying :func:`~BaseStep.update_hyperparams_space` to all steps. @@ -1768,7 +1768,7 @@ def update_hyperparams(self, hyperparams: HyperparameterSamples) -> 'BaseStep': self.apply(method='_update_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) return self - def update_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> 'BaseStep': + def update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> 'BaseStep': """ Update hyperparams space recursively by applying :func:`~BaseStep.update_hyperparams_space` to all steps. @@ -1806,7 +1806,7 @@ def get_hyperparams(self) -> HyperparameterSamples: results: HyperparameterSamples = self.apply(method='_get_hyperparams', ra=_RecursiveArguments()) return results.to_flat() - def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: + def set_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> BaseStep: """ Set all the pipeline hyperparams by applying :func:`~BaseStep.set_hyperparams` to all steps. @@ -1820,7 +1820,7 @@ def set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: self.apply(method='_set_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) return self - def set_params(self, hyperparams: HyperparameterSamples) -> BaseStep: + def set_params(self, hyperparams: Union[Dict, HyperparameterSamples]) -> BaseStep: """ Set all the pipeline hyperparams by applying :func:`~BaseStep.set_hyperparams` to all steps. @@ -1834,7 +1834,7 @@ def set_params(self, hyperparams: HyperparameterSamples) -> BaseStep: self.apply(method='_set_params', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) return self - def set_hyperparams_space(self, hyperparams_space: HyperparameterSpace) -> BaseStep: + def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> BaseStep: """ Set all the pipeline hyperparams space by applying :func:`~BaseStep.set_hyperparams_space` to all steps. diff --git a/testing/hyperparams/test_get_set_hyperparams.py b/testing/hyperparams/test_get_set_hyperparams.py index c6b6aa4b..49d33783 100644 --- a/testing/hyperparams/test_get_set_hyperparams.py +++ b/testing/hyperparams/test_get_set_hyperparams.py @@ -41,10 +41,10 @@ def inverse_transform(self, processed_outputs): def test_step_cloner_should_get_hyperparams(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) hyperparams = p.get_hyperparams() @@ -55,10 +55,10 @@ def test_step_cloner_should_get_hyperparams(): def test_step_cloner_should_set_hyperparams(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams[META_STEP_HP] == META_STEP_HP_VALUE @@ -67,14 +67,14 @@ def test_step_cloner_should_set_hyperparams(): def test_step_cloner_update_hyperparams_should_update_step_cloner_hyperparams(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) - p.update_hyperparams(HyperparameterSamples({ + p.update_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE + 1, - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams[META_STEP_HP] == META_STEP_HP_VALUE + 1 @@ -83,14 +83,14 @@ def test_step_cloner_update_hyperparams_should_update_step_cloner_hyperparams(): def test_step_cloner_update_hyperparams_should_update_wrapped_step_hyperparams(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) - p.update_hyperparams(HyperparameterSamples({ + p.update_hyperparams({ SOME_STEP_HP: SOME_STEP_HP_VALUE + 1 - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams[META_STEP_HP] == META_STEP_HP_VALUE @@ -100,15 +100,15 @@ def test_step_cloner_update_hyperparams_should_update_wrapped_step_hyperparams() def test_step_cloner_update_hyperparams_space_should_update_step_cloner_hyperparams(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ META_STEP_HP: RAND_INT_META_STEP, SOME_STEP_HP: RAND_INT_SOME_STEP - })) + }) update_meta_step_hp_space = RandInt(0, 40) - p.update_hyperparams_space(HyperparameterSpace({ + p.update_hyperparams_space({ META_STEP_HP: update_meta_step_hp_space - })) + }) assert isinstance(p.hyperparams_space, HyperparameterSpace) assert p.hyperparams_space[META_STEP_HP] == update_meta_step_hp_space @@ -123,9 +123,9 @@ def test_step_cloner_update_hyperparams_space_should_update_wrapped_step_hyperpa })) updated_some_step_hp_space = RandInt(0, 400) - p.update_hyperparams_space(HyperparameterSpace({ + p.update_hyperparams_space({ SOME_STEP_HP: updated_some_step_hp_space - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams_space[META_STEP_HP] == RAND_INT_META_STEP @@ -135,10 +135,10 @@ def test_step_cloner_update_hyperparams_space_should_update_wrapped_step_hyperpa def test_step_cloner_should_set_steps_hyperparams(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert isinstance(p.get_step().hyperparams, HyperparameterSamples) @@ -148,10 +148,10 @@ def test_step_cloner_should_set_steps_hyperparams(): def test_step_cloner_should_set_steps_hyperparams_space(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ META_STEP_HP: RAND_INT_STEP_CLONER, SOME_STEP_HP: RAND_INT_SOME_STEP - })) + }) assert isinstance(p.get_step().hyperparams_space, HyperparameterSpace) assert p.get_step().hyperparams_space[SOME_STEP_HP_KEY] == RAND_INT_SOME_STEP @@ -160,10 +160,10 @@ def test_step_cloner_should_set_steps_hyperparams_space(): def test_step_cloner_should_set_hyperparams_space(): p = StepClonerForEachDataInput(SomeStep()) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ META_STEP_HP: RAND_INT_STEP_CLONER, SOME_STEP_HP: RAND_INT_SOME_STEP - })) + }) assert isinstance(p.hyperparams_space, HyperparameterSpace) assert p.hyperparams_space[META_STEP_HP] == RAND_INT_STEP_CLONER @@ -192,11 +192,11 @@ def test_pipeline_should_set_hyperparams(): SomeStep().set_name('step_2') ]) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ 'hp': 1, 'step_1__hp': 2, 'step_2__hp': 3 - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams['hp'] == 1 @@ -209,11 +209,11 @@ def test_pipeline_should_get_hyperparams(): SomeStep().set_name('step_1'), SomeStep().set_name('step_2') ]) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ 'hp': 1, 'step_1__hp': 2, 'step_2__hp': 3 - })) + }) hyperparams = p.get_hyperparams() @@ -228,11 +228,11 @@ def test_pipeline_should_get_hyperparams_space(): SomeStep().set_name('step_1'), SomeStep().set_name('step_2') ]) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ 'hp': RandInt(1, 2), 'step_1__hp': RandInt(2, 3), 'step_2__hp': RandInt(3, 4) - })) + }) hyperparams_space = p.get_hyperparams_space() @@ -254,16 +254,16 @@ def test_pipeline_should_update_hyperparams(): SomeStep().set_name('step_2') ]) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ 'hp': 1, 'step_1__hp': 2, 'step_2__hp': 3 - })) + }) - p.update_hyperparams(HyperparameterSamples({ + p.update_hyperparams({ 'hp': 4, 'step_2__hp': 6 - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams['hp'] == 4 @@ -277,15 +277,15 @@ def test_pipeline_should_update_hyperparams_space(): SomeStep().set_name('step_2') ]) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ 'hp': RandInt(1, 2), 'step_1__hp': RandInt(2, 3), 'step_2__hp': RandInt(3, 4) - })) - p.update_hyperparams_space(HyperparameterSpace({ + }) + p.update_hyperparams_space({ 'hp': RandInt(4, 6), 'step_2__hp': RandInt(6, 8) - })) + }) assert isinstance(p.hyperparams_space, HyperparameterSpace) @@ -301,10 +301,10 @@ def test_pipeline_should_update_hyperparams_space(): def test_meta_step_mixin_should_get_hyperparams(): p = SomeMetaStepMixin(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) hyperparams = p.get_hyperparams() @@ -315,10 +315,10 @@ def test_meta_step_mixin_should_get_hyperparams(): def test_meta_step_mixin_should_set_hyperparams(): p = SomeMetaStepMixin(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) assert isinstance(p.hyperparams, HyperparameterSamples) assert p.hyperparams[META_STEP_HP] == META_STEP_HP_VALUE @@ -327,14 +327,14 @@ def test_meta_step_mixin_should_set_hyperparams(): def test_meta_step_mixin_update_hyperparams_should_update_meta_step_hyperparams(): p = SomeMetaStepMixin(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) - p.update_hyperparams(HyperparameterSamples({ + p.update_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE + 1 - })) + }) assert p.hyperparams[META_STEP_HP] == META_STEP_HP_VALUE + 1 assert p.get_step().get_hyperparams()['somestep_hyperparam'] == SOME_STEP_HP_VALUE @@ -342,14 +342,14 @@ def test_meta_step_mixin_update_hyperparams_should_update_meta_step_hyperparams( def test_meta_step_mixin_update_hyperparams_should_update_wrapped_step_hyperparams(): p = SomeMetaStepMixin(SomeStep()) - p.set_hyperparams(HyperparameterSamples({ + p.set_hyperparams({ META_STEP_HP: META_STEP_HP_VALUE, SOME_STEP_HP: SOME_STEP_HP_VALUE - })) + }) - p.update_hyperparams(HyperparameterSamples({ + p.update_hyperparams({ SOME_STEP_HP: SOME_STEP_HP_VALUE + 1 - })) + }) assert p.hyperparams[META_STEP_HP] == META_STEP_HP_VALUE assert p.get_step().get_hyperparams()['somestep_hyperparam'] == SOME_STEP_HP_VALUE + 1 @@ -363,9 +363,9 @@ def test_meta_step_mixin_update_hyperparams_space_should_update_meta_step_hyperp })) update_meta_step_hp_space = RandInt(0, 100) - p.update_hyperparams_space(HyperparameterSpace({ + p.update_hyperparams_space({ META_STEP_HP: update_meta_step_hp_space - })) + }) assert p.hyperparams_space[META_STEP_HP] == update_meta_step_hp_space assert p.wrapped.get_hyperparams_space()['somestep_hyperparam'] == RAND_INT_SOME_STEP @@ -373,15 +373,15 @@ def test_meta_step_mixin_update_hyperparams_space_should_update_meta_step_hyperp def test_meta_step_mixin_update_hyperparams_space_should_update_wrapped_step_hyperparams(): p = SomeMetaStepMixin(SomeStep()) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ META_STEP_HP: RAND_INT_META_STEP, SOME_STEP_HP: RAND_INT_SOME_STEP - })) + }) updated_some_step_hp_space = RandInt(0, 100) - p.update_hyperparams_space(HyperparameterSpace({ + p.update_hyperparams_space({ SOME_STEP_HP: updated_some_step_hp_space - })) + }) assert p.hyperparams_space[META_STEP_HP] == RAND_INT_META_STEP assert p.wrapped.get_hyperparams_space()['somestep_hyperparam'] == updated_some_step_hp_space @@ -389,10 +389,10 @@ def test_meta_step_mixin_update_hyperparams_space_should_update_wrapped_step_hyp def test_meta_step_mixin_should_set_hyperparams_space(): p = SomeMetaStepMixin(SomeStep()) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ META_STEP_HP: RAND_INT_META_STEP, SOME_STEP_HP: RAND_INT_SOME_STEP - })) + }) assert p.hyperparams_space[META_STEP_HP] == RAND_INT_META_STEP assert p.get_step().hyperparams_space[SOME_STEP_HP_KEY] == RAND_INT_SOME_STEP @@ -400,10 +400,10 @@ def test_meta_step_mixin_should_set_hyperparams_space(): def test_meta_step_mixin_should_get_hyperparams_space(): p = SomeMetaStepMixin(SomeStep()) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ META_STEP_HP: RAND_INT_META_STEP, SOME_STEP_HP: RAND_INT_SOME_STEP - })) + }) hyperparams_space = p.get_hyperparams_space() From 48dbf84fac11e6f03653b54e09bef2cfe0d6362e Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:31:49 -0400 Subject: [PATCH 38/48] Complete Docstring For get_children --- neuraxle/base.py | 8 ++++++-- testing/steps/test_step_cloner_for_each_data_input.py | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index e5380674..8c623c68 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -2057,9 +2057,13 @@ def resume(self, data_container: DataContainer, context: ExecutionContext): def get_children(self) -> List[BaseStep]: """ - Get the list of all the children for that step. + Get the list of all the childs for that step. + :class:`_HasChildrenMixin` calls this method to apply methods to all of the childs for that step. - :return: + :return: list of child steps + + .. seealso:: + :class:`_HasChildrenMixin` """ return [self.wrapped] diff --git a/testing/steps/test_step_cloner_for_each_data_input.py b/testing/steps/test_step_cloner_for_each_data_input.py index 7c335d6b..896db560 100644 --- a/testing/steps/test_step_cloner_for_each_data_input.py +++ b/testing/steps/test_step_cloner_for_each_data_input.py @@ -159,7 +159,3 @@ def test_step_cloner_should_load_sub_steps(tmpdir): def _create_data(shape): data_inputs = np.random.random(shape).astype(np.float32) return data_inputs - - -def test_step_cloner_should_set_train(tmpdir): - pass From 924d922a7946d6b0fa3b62778dc50df2edd3e418 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:35:24 -0400 Subject: [PATCH 39/48] Remove ra from recursive methods in _HasChildrenMixin --- neuraxle/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 8c623c68..80efe869 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1765,7 +1765,7 @@ def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> """ if not isinstance(hyperparams, RecursiveDict): hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_update_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) + self.apply(method='_update_hyperparams', hyperparams=hyperparams.to_flat()) return self def update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> 'BaseStep': @@ -1779,7 +1779,7 @@ def update_hyperparams_space(self, hyperparams_space: Union[Dict, Hyperparameter """ if not isinstance(hyperparams_space, RecursiveDict): hyperparams_space = HyperparameterSpace(hyperparams_space) - self.apply(method='_update_hyperparams_space', ra=_RecursiveArguments(hyperparams_space=hyperparams_space.to_flat())) + self.apply(method='_update_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) return self def get_hyperparams_space(self) -> HyperparameterSpace: @@ -1791,7 +1791,7 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :func:`~BaseStep.get_hyperparams_space`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - results: HyperparameterSpace = self.apply(method='_get_hyperparams_space', ra=_RecursiveArguments()) + results: HyperparameterSpace = self.apply(method='_get_hyperparams_space') return results.to_flat() def get_hyperparams(self) -> HyperparameterSamples: @@ -1803,7 +1803,7 @@ def get_hyperparams(self) -> HyperparameterSamples: :func:`~BaseStep.get_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - results: HyperparameterSamples = self.apply(method='_get_hyperparams', ra=_RecursiveArguments()) + results: HyperparameterSamples = self.apply(method='_get_hyperparams') return results.to_flat() def set_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> BaseStep: @@ -1817,7 +1817,7 @@ def set_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> Ba """ if not isinstance(hyperparams, RecursiveDict): hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_set_hyperparams', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) + self.apply(method='_set_hyperparams', hyperparams=hyperparams.to_flat()) return self def set_params(self, hyperparams: Union[Dict, HyperparameterSamples]) -> BaseStep: @@ -1831,7 +1831,7 @@ def set_params(self, hyperparams: Union[Dict, HyperparameterSamples]) -> BaseSte """ if not isinstance(hyperparams, RecursiveDict): hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_set_params', ra=_RecursiveArguments(hyperparams=hyperparams.to_flat())) + self.apply(method='_set_params', hyperparams=hyperparams.to_flat()) return self def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> BaseStep: @@ -1845,7 +1845,7 @@ def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpa """ if not isinstance(hyperparams_space, RecursiveDict): hyperparams_space = HyperparameterSpace(hyperparams_space) - self.apply(method='_set_hyperparams_space', ra=_RecursiveArguments(hyperparams_space=hyperparams_space.to_flat())) + self.apply(method='_set_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) return self def set_train(self, is_train: bool = True) -> BaseStep: @@ -1859,7 +1859,7 @@ def set_train(self, is_train: bool = True) -> BaseStep: .. seealso:: :func:`BaseStep.set_train` """ - self.apply(method='_set_train', ra=_RecursiveArguments(is_train=is_train)) + self.apply(method='_set_train', is_train=is_train) return self From a2984f90c4820f433ebfacfd8cc2832fe34de472 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:50:42 -0400 Subject: [PATCH 40/48] Move Recursive Methods to BaseStep --- neuraxle/base.py | 148 ++++++++++------------------------------------- 1 file changed, 29 insertions(+), 119 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 80efe869..f40b49ec 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -749,7 +749,8 @@ def set_train(self, is_train: bool = True): :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._set_train` """ - return self._set_train(is_train) + self.apply(method='_set_train', is_train=is_train) + return self def _set_train(self, is_train): self.is_train = is_train @@ -826,7 +827,10 @@ def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> 'B :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._set_train` """ - return self._set_hyperparams(hyperparams) + if not isinstance(hyperparams, RecursiveDict): + hyperparams = HyperparameterSamples(hyperparams) + self.apply(method='_set_hyperparams', hyperparams=hyperparams.to_flat()) + return self def _set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]): self.invalidate() @@ -867,8 +871,9 @@ def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._update_hyperparams` """ - self._update_hyperparams(hyperparams) - + if not isinstance(hyperparams, RecursiveDict): + hyperparams = HyperparameterSamples(hyperparams) + self.apply(method='_update_hyperparams', hyperparams=hyperparams.to_flat()) return self def _update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]): @@ -889,7 +894,8 @@ def get_hyperparams(self) -> HyperparameterSamples: :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._get_hyperparams` """ - return self._get_hyperparams() + results: HyperparameterSamples = self.apply(method='_get_hyperparams') + return results.to_flat() def _get_hyperparams(self): return HyperparameterSamples(self.hyperparams.to_flat_as_dict_primitive()) @@ -917,12 +923,15 @@ def set_params(self, **params) -> 'BaseStep': :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._set_params` """ - return self._set_params(params) + if not isinstance(params, RecursiveDict): + params = HyperparameterSamples(params) + self.apply(method='_set_params', params=params.to_flat()) + return self def _set_params(self, params: dict): return self.set_hyperparams(HyperparameterSamples(params)) - def get_params(self) -> dict: + def get_params(self) -> HyperparameterSamples: """ Get step hyperparameters as a flat primitive dict. @@ -945,7 +954,8 @@ def get_params(self) -> dict: :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._get_params` """ - return self._get_params() + results: HyperparameterSamples = self.apply(method='_get_params') + return results def _get_params(self): return self.get_hyperparams().to_flat_as_ordered_dict_primitive() @@ -978,7 +988,10 @@ def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpa :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._get_params` """ - return self._set_hyperparams_space(hyperparams_space) + if not isinstance(hyperparams_space, RecursiveDict): + hyperparams_space = HyperparameterSpace(hyperparams_space) + self.apply(method='_set_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) + return self def _set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): self.invalidate() @@ -1015,7 +1028,10 @@ def update_hyperparams_space(self, hyperparams_space: Union[Dict, Hyperparameter :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - return self._update_hyperparams_space(hyperparams_space) + if not isinstance(hyperparams_space, RecursiveDict): + hyperparams_space = HyperparameterSpace(hyperparams_space) + self.apply(method='_update_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) + return self def _update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): self.invalidate() @@ -1040,7 +1056,8 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` """ - return self._get_hyperparams_space() + results: HyperparameterSpace = self.apply(method='_get_hyperparams_space') + return results.to_flat() def _get_hyperparams_space(self): return HyperparameterSpace(self.hyperparams_space.to_flat_as_dict_primitive()) @@ -1096,7 +1113,7 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a try: return method(*kargs, **ra.kwargs) - except Exception: + except Exception as err: print('{}: Failed to apply method {}.'.format(self.name, method)) print(traceback.format_stack()) @@ -1754,113 +1771,6 @@ def get_children(self) -> List[BaseStep]: """ pass - def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> 'BaseStep': - """ - Update hyperparams recursively by applying :func:`~BaseStep.update_hyperparams_space` to all steps. - - .. seealso:: - :func:`~BaseStep.apply`, - :func:`~BaseStep.update_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSpace` - """ - if not isinstance(hyperparams, RecursiveDict): - hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_update_hyperparams', hyperparams=hyperparams.to_flat()) - return self - - def update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> 'BaseStep': - """ - Update hyperparams space recursively by applying :func:`~BaseStep.update_hyperparams_space` to all steps. - - .. seealso:: - :func:`~BaseStep.apply`, - :func:`~BaseStep.update_hyperparams_space`, - :class:`~neuraxle.hyperparams.space.HyperparameterSpace` - """ - if not isinstance(hyperparams_space, RecursiveDict): - hyperparams_space = HyperparameterSpace(hyperparams_space) - self.apply(method='_update_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) - return self - - def get_hyperparams_space(self) -> HyperparameterSpace: - """ - Get all the pipeline hyperparams by applying :func:`~BaseStep.get_hyperparams_space` to all steps. - - .. seealso:: - :func:`~BaseStep.apply`, - :func:`~BaseStep.get_hyperparams_space`, - :class:`~neuraxle.hyperparams.space.HyperparameterSpace` - """ - results: HyperparameterSpace = self.apply(method='_get_hyperparams_space') - return results.to_flat() - - def get_hyperparams(self) -> HyperparameterSamples: - """ - Get all the pipeline hyperparams by applying :func:`~BaseStep.get_hyperparams` to all steps. - - .. seealso:: - :func:`~BaseStep.apply`, - :func:`~BaseStep.get_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSpace` - """ - results: HyperparameterSamples = self.apply(method='_get_hyperparams') - return results.to_flat() - - def set_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> BaseStep: - """ - Set all the pipeline hyperparams by applying :func:`~BaseStep.set_hyperparams` to all steps. - - .. seealso:: - :func:`~BaseStep.apply`, - :func:`~BaseStep.set_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - if not isinstance(hyperparams, RecursiveDict): - hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_set_hyperparams', hyperparams=hyperparams.to_flat()) - return self - - def set_params(self, hyperparams: Union[Dict, HyperparameterSamples]) -> BaseStep: - """ - Set all the pipeline hyperparams by applying :func:`~BaseStep.set_hyperparams` to all steps. - - .. seealso:: - :func:`~BaseStep.apply`, - :func:`~BaseStep.set_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSamples` - """ - if not isinstance(hyperparams, RecursiveDict): - hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_set_params', hyperparams=hyperparams.to_flat()) - return self - - def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> BaseStep: - """ - Set all the pipeline hyperparams space by applying :func:`~BaseStep.set_hyperparams_space` to all steps. - - .. seealso:: - :func:`~BaseStep.apply`, - :func:`~BaseStep.set_hyperparams`, - :class:`~neuraxle.hyperparams.space.HyperparameterSpace` - """ - if not isinstance(hyperparams_space, RecursiveDict): - hyperparams_space = HyperparameterSpace(hyperparams_space) - self.apply(method='_set_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) - return self - - def set_train(self, is_train: bool = True) -> BaseStep: - """ - This method overrides the method of BaseStep to also consider the wrapped step as well as self. - Set pipeline step mode to train or test. - - :param is_train: is training mode or not - :return: - - .. seealso:: - :func:`BaseStep.set_train` - """ - self.apply(method='_set_train', is_train=is_train) - return self class MetaStepMixin(_HasChildrenMixin): From fdd97b5448fdffbb45e942bda462382914a55b62 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 14:55:32 -0400 Subject: [PATCH 41/48] Fix Notes For Recursive Methods --- neuraxle/base.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index f40b49ec..7eb247aa 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -819,7 +819,7 @@ def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> 'B :return: self .. note:: - This is a recursive method. + This is a recursive method that will call :func:`BaseStep._set_hyperparams` in the end. .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, :class:̀_HasChildrenMixin`, @@ -863,7 +863,7 @@ def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> :return: self .. note:: - This is a recursive method. + This is a recursive method that will call :func:`BaseStep._update_hyperparams` in the end. .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, :class:̀_HasChildrenMixin`, @@ -886,7 +886,7 @@ def get_hyperparams(self) -> HyperparameterSamples: :return: step hyperparameters .. note:: - This is a recursive method. + This is a recursive method that will call :func:`BaseStep._get_hyperparams` in the end. .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, :class:̀_HasChildrenMixin`, @@ -915,7 +915,7 @@ def set_params(self, **params) -> 'BaseStep': :param **params: arbitrary number of arguments for hyperparameters .. note:: - This is a recursive method. + This is a recursive method that will call :func:`BaseStep._set_params` in the end. .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, :class:̀_HasChildrenMixin`, @@ -946,7 +946,7 @@ def get_params(self) -> HyperparameterSamples: :return: hyperparameters .. note:: - This is a recursive method. + This is a recursive method that will call :func:`BaseStep._get_params` in the end. .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, :class:̀_HasChildrenMixin`, @@ -980,7 +980,7 @@ def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpa :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` .. note:: - This is a recursive method. + This is a recursive method that will call :func:`BaseStep._set_hyperparams_space` in the end. .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSamples`, :class:̀_HasChildrenMixin`, @@ -1024,6 +1024,8 @@ def update_hyperparams_space(self, hyperparams_space: Union[Dict, Hyperparameter :param hyperparams_space: hyperparameters space :return: self + .. note:: + This is a recursive method that will call :func:`BaseStep._update_hyperparams_space` in the end. .. seealso:: :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` @@ -1052,6 +1054,8 @@ def get_hyperparams_space(self) -> HyperparameterSpace: :return: step hyperparams space + .. note:: + This is a recursive method that will call :func:`BaseStep._get_hyperparams_space` in the end. .. seealso:: :class:`~neuraxle.hyperparams.space.HyperparameterSpace`, :class:`~neuraxle.hyperparams.distributions.HyperparameterDistribution` From f18dd8b9ae479938f947ec03f123dd892da5e72d Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 15:00:25 -0400 Subject: [PATCH 42/48] Fix Casting In BaseStep Recursive Methods --- neuraxle/base.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 7eb247aa..49f62dfb 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -827,9 +827,7 @@ def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> 'B :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._set_train` """ - if not isinstance(hyperparams, RecursiveDict): - hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_set_hyperparams', hyperparams=hyperparams.to_flat()) + self.apply(method='_set_hyperparams', hyperparams=HyperparameterSamples(hyperparams).to_flat()) return self def _set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]): @@ -871,9 +869,7 @@ def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._update_hyperparams` """ - if not isinstance(hyperparams, RecursiveDict): - hyperparams = HyperparameterSamples(hyperparams) - self.apply(method='_update_hyperparams', hyperparams=hyperparams.to_flat()) + self.apply(method='_update_hyperparams', hyperparams=HyperparameterSamples(hyperparams).to_flat()) return self def _update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]): @@ -923,9 +919,7 @@ def set_params(self, **params) -> 'BaseStep': :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._set_params` """ - if not isinstance(params, RecursiveDict): - params = HyperparameterSamples(params) - self.apply(method='_set_params', params=params.to_flat()) + self.apply(method='_set_params', params=HyperparameterSamples(params).to_flat()) return self def _set_params(self, params: dict): @@ -988,9 +982,7 @@ def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpa :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._get_params` """ - if not isinstance(hyperparams_space, RecursiveDict): - hyperparams_space = HyperparameterSpace(hyperparams_space) - self.apply(method='_set_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) + self.apply(method='_set_hyperparams_space', hyperparams_space=HyperparameterSpace(hyperparams_space).to_flat()) return self def _set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): @@ -1030,9 +1022,7 @@ def update_hyperparams_space(self, hyperparams_space: Union[Dict, Hyperparameter :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - if not isinstance(hyperparams_space, RecursiveDict): - hyperparams_space = HyperparameterSpace(hyperparams_space) - self.apply(method='_update_hyperparams_space', hyperparams_space=hyperparams_space.to_flat()) + self.apply(method='_update_hyperparams_space', hyperparams_space=HyperparameterSpace(hyperparams_space).to_flat()) return self def _update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): From 8b233127554f9c498b0dd62ddf2a10b5d06c8c94 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 17:38:15 -0400 Subject: [PATCH 43/48] Allow different separators at each level inside a Recursive Dict --- examples/hyperparams/plot_hyperparams.py | 4 +- neuraxle/base.py | 42 ++++----- neuraxle/hyperparams/space.py | 103 ++++++++++++----------- neuraxle/steps/flow.py | 8 +- neuraxle/steps/loop.py | 2 +- testing/hyperparams/test_space.py | 7 +- testing/test_optional.py | 8 +- testing/test_pickle_checkpoint_step.py | 3 +- testing/test_pipeline.py | 6 +- testing/test_recursive_dict.py | 50 +++++++++-- 10 files changed, 138 insertions(+), 95 deletions(-) diff --git a/examples/hyperparams/plot_hyperparams.py b/examples/hyperparams/plot_hyperparams.py index 6fc34759..a7459348 100644 --- a/examples/hyperparams/plot_hyperparams.py +++ b/examples/hyperparams/plot_hyperparams.py @@ -45,11 +45,11 @@ def main(): ]) ]) - p.set_hyperparams_space(HyperparameterSpace({ + p.set_hyperparams_space({ 'step1__multiply_by': RandInt(42, 50), 'step2__multiply_by': RandInt(-10, 0), 'Pipeline__PCA__n_components': RandInt(2, 3) - })) + }) samples = p.get_hyperparams_space().rvs() p.set_hyperparams(samples) diff --git a/neuraxle/base.py b/neuraxle/base.py index 49f62dfb..2baed25f 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -644,7 +644,7 @@ def __init__( self.pending_mutate: ('BaseStep', str, str) = (None, None, None) self.is_initialized = False - self.invalidate() + self._invalidate() self.is_train: bool = True def summary_hash(self, data_container: DataContainer) -> str: @@ -718,7 +718,8 @@ def invalidate(self) -> 'BaseStep': :func:`BaseStep.apply`, :func:`_HasChildrenMixin._apply` """ - return self._invalidate() + self.apply(method='_invalidate') + return self def _invalidate(self): self.is_invalidated = True @@ -767,7 +768,7 @@ def set_name(self, name: str): A step name is the same value as the one in the keys of :py:attr:`~neuraxle.pipeline.Pipeline.steps_as_tuple` """ self.name = name - self.invalidate() + self._invalidate() return self def get_name(self) -> str: @@ -831,7 +832,7 @@ def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> 'B return self def _set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]): - self.invalidate() + self._invalidate() hyperparams = HyperparameterSamples(hyperparams).to_flat() self.hyperparams = hyperparams if len(hyperparams) > 0 else self.hyperparams return self @@ -982,11 +983,12 @@ def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpa :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._get_params` """ - self.apply(method='_set_hyperparams_space', hyperparams_space=HyperparameterSpace(hyperparams_space).to_flat()) + self.apply(method='_set_hyperparams_space', hyperparams_space=HyperparameterSpace( + hyperparams_space).to_flat()) return self def _set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): - self.invalidate() + self._invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() self.hyperparams_space = HyperparameterSpace(hyperparams_space) if len( hyperparams_space) > 0 else self.hyperparams_space @@ -1022,11 +1024,12 @@ def update_hyperparams_space(self, hyperparams_space: Union[Dict, Hyperparameter :func:`~BaseStep.update_hyperparams`, :class:`~neuraxle.hyperparams.space.HyperparameterSpace` """ - self.apply(method='_update_hyperparams_space', hyperparams_space=HyperparameterSpace(hyperparams_space).to_flat()) + self.apply(method='_update_hyperparams_space', hyperparams_space=HyperparameterSpace( + hyperparams_space).to_flat()) return self def _update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): - self.invalidate() + self._invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() self.hyperparams_space.update(HyperparameterSpace(hyperparams_space).to_flat()) return self @@ -1201,7 +1204,7 @@ def _will_fit(self, data_container: DataContainer, context: ExecutionContext) -> :param context: execution context :return: (data container, execution context) """ - self.invalidate() + self._invalidate() return data_container, context.push(self) def _did_fit(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: @@ -1233,7 +1236,7 @@ def _will_fit_transform(self, data_container: DataContainer, context: ExecutionC :param context: execution context :return: (data container, execution context) """ - self.invalidate() + self._invalidate() return data_container, context.push(self) def _did_fit_transform(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: @@ -1343,7 +1346,7 @@ def fit_transform(self, data_inputs, expected_outputs=None) -> ('BaseStep', Any) :param expected_outputs: expected outputs to fit on :return: (fitted self, tranformed data inputs) """ - self.invalidate() + self._invalidate() new_self = self.fit(data_inputs, expected_outputs) out = new_self.transform(data_inputs) @@ -1456,7 +1459,7 @@ def _initialize_if_needed(step): return step def _invalidate(step): - step.invalidate() + step._invalidate() if full_dump: # initialize and invalidate steps to make sure that all steps will be saved @@ -1548,7 +1551,7 @@ def mutate(self, new_method="inverse_transform", method_to_assign_to="transform" :param warn: (verbose) wheter or not to warn about the inexistence of the method. :return: self, a copy of self, or even perhaps a new or different BaseStep object. """ - self.invalidate() + self._invalidate() pending_new_base_step, pending_new_method, pending_method_to_assign_to = self.pending_mutate # Use everything that is pending if they are not none (ternaries). @@ -1569,7 +1572,7 @@ def mutate(self, new_method="inverse_transform", method_to_assign_to="transform" # 3. assign new method to old method setattr(new_base_step, method_to_assign_to, new_method) - self.invalidate() + self._invalidate() except AttributeError as e: if warn: @@ -1613,7 +1616,7 @@ def will_mutate_to( :param new_method: if it is not None, upon calling ``mutate``, the new_method will be the one that is used on the provided new_base_step. :return: self """ - self.invalidate() + self._invalidate() if new_method is None or method_to_assign_to is None: new_method = method_to_assign_to = "transform" # No changes will be applied (transform will stay transform). @@ -1766,7 +1769,6 @@ def get_children(self) -> List[BaseStep]: pass - class MetaStepMixin(_HasChildrenMixin): """ A class to represent a step that wraps another step. It can be used for many things. @@ -1843,7 +1845,7 @@ def set_step(self, step: BaseStep) -> BaseStep: :param step: new wrapped step :return: self """ - self.invalidate() + self._invalidate() self.wrapped: BaseStep = _sklearn_to_neuraxle_step(step) return self @@ -2385,7 +2387,7 @@ def _patch_missing_names(self, steps_as_tuple: NamedTupleList) -> NamedTupleList step = (_name, step) names_yet.add(step[0]) patched.append(step) - self.invalidate() + self._invalidate() return patched def _rename_step(self, step_name, class_name, names_yet: set): @@ -2403,7 +2405,7 @@ def _rename_step(self, step_name, class_name, names_yet: set): while step_name in names_yet: step_name = class_name + str(i) i += 1 - self.invalidate() + self._invalidate() return step_name def _refresh_steps(self): @@ -2411,7 +2413,7 @@ def _refresh_steps(self): Private method to refresh inner state after having edited ``self.steps_as_tuple`` (recreate ``self.steps`` from ``self.steps_as_tuple``). """ - self.invalidate() + self._invalidate() self.steps: OrderedDict = OrderedDict(self.steps_as_tuple) for name, step in self.items(): step.name = name diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index 162cb6ed..d2f34f06 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -66,8 +66,6 @@ from neuraxle.hyperparams.distributions import HyperparameterDistribution -PARAMS_SPLIT_SEQ = "__" - class RecursiveDict(OrderedDict): """ @@ -79,7 +77,7 @@ class RecursiveDict(OrderedDict): """ DEFAULT_SEPARATOR = '__' - def __init__(self, separator=None, *args, **kwds): + def __init__(self, *args, separator=None, **kwds): if len(args) == 1 and isinstance(args[0], RecursiveDict) and len(kwds) == 0: super().__init__(args[0].items()) separator = args[0].separator @@ -114,7 +112,7 @@ def to_flat(self) -> 'RecursiveDict': :return: an HyperparameterSamples like self, flattened. """ - return nested_dict_to_flat(self, dict_ctor=type(self)) + return self.nested_dict_to_flat(dict_ctor=type(self)) def to_nested_dict(self) -> 'RecursiveDict': """ @@ -122,7 +120,7 @@ def to_nested_dict(self) -> 'RecursiveDict': :return: an HyperparameterSamples like self, as a nested dict. """ - return flat_to_nested_dict(self, dict_ctor=type(self)) + return self.flat_to_nested_dict(dict_ctor=type(self)) def to_flat_as_dict_primitive(self) -> dict: """ @@ -130,7 +128,7 @@ def to_flat_as_dict_primitive(self) -> dict: :return: an HyperparameterSpace like self, flattened. """ - return nested_dict_to_flat(self, dict_ctor=dict) + return self.nested_dict_to_flat(dict_ctor=dict) def to_nested_dict_as_dict_primitive(self) -> dict: """ @@ -138,7 +136,7 @@ def to_nested_dict_as_dict_primitive(self) -> dict: :return: a nested primitive dict type of self. """ - return flat_to_nested_dict(self, dict_ctor=dict) + return self.flat_to_nested_dict(dict_ctor=dict) def to_flat_as_ordered_dict_primitive(self) -> OrderedDict: """ @@ -146,7 +144,7 @@ def to_flat_as_ordered_dict_primitive(self) -> OrderedDict: :return: an HyperparameterSpace like self, flattened. """ - return nested_dict_to_flat(self, dict_ctor=OrderedDict) + return self.nested_dict_to_flat(dict_ctor=OrderedDict) def to_nested_dict_as_ordered_dict_primitive(self) -> OrderedDict: """ @@ -154,49 +152,52 @@ def to_nested_dict_as_ordered_dict_primitive(self) -> OrderedDict: :return: a nested primitive dict type of self. """ - return flat_to_nested_dict(self, dict_ctor=OrderedDict) - - -def nested_dict_to_flat(nested_dict, dict_ctor=OrderedDict): - """ - Convert a nested hyperparameter dictionary to a flat one. + return self.flat_to_nested_dict(dict_ctor=OrderedDict) - :param nested_dict: a nested hyperparameter dictionary. - :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. - :return: a flat hyperparameter dictionary. - """ - ret = dict_ctor() - for k, v in nested_dict.items(): - if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor): - _ret = nested_dict_to_flat(v, dict_ctor) - for key, val in _ret.items(): - ret[k + PARAMS_SPLIT_SEQ + key] = val - else: - ret[k] = v - return ret + def nested_dict_to_flat(self, dict_ctor): + """ + Convert a nested hyperparameter dictionary to a flat one. + :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. + :return: a flat hyperparameter dictionary. + """ -def flat_to_nested_dict(flat_dict, dict_ctor=dict): - """ - Convert a flat hyperparameter dictionary to a nested one. + ret = dict_ctor() + for k, v in self.items(): + if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor): + nested_dict = v + if not isinstance(v, RecursiveDict): + nested_dict = RecursiveDict(v) + _ret = nested_dict.nested_dict_to_flat(dict_ctor=type(nested_dict)) + for key, val in _ret.items(): + ret[k + nested_dict.separator + key] = val + else: + ret[k] = v + return ret + + def flat_to_nested_dict(self, dict_ctor=dict): + """ + Convert a flat hyperparameter dictionary to a nested one. - :param flat_dict: a flat hyperparameter dictionary. - :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. - :return: a nested hyperparameter dictionary. - """ - pre_ret = dict_ctor() - ret = dict_ctor() - for k, v in flat_dict.items(): - k, _, key = k.partition(PARAMS_SPLIT_SEQ) - if len(key) > 0: - if k not in pre_ret.keys(): - pre_ret[k] = dict_ctor() - pre_ret[k][key] = v - else: - ret[k] = v - for k, v in pre_ret.items(): - ret[k] = flat_to_nested_dict(v, dict_ctor) - return ret + :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. + :return: a nested hyperparameter dictionary. + """ + pre_ret = dict_ctor() + ret = dict_ctor() + for k, v in self.items(): + k, _, key = k.partition(self.separator) + if len(key) > 0: + if k not in pre_ret.keys(): + pre_ret[k] = dict_ctor() + pre_ret[k][key] = v + else: + ret[k] = v + for k, v in pre_ret.items(): + flat_dict = v + if not isinstance(v, RecursiveDict): + flat_dict = RecursiveDict(v) + ret[k] = flat_dict.flat_to_nested_dict(dict_ctor=dict_ctor) + return ret class HyperparameterSamples(RecursiveDict): @@ -208,8 +209,8 @@ class HyperparameterSamples(RecursiveDict): HyperparameterSamples are often the result of calling ``.rvs()`` on an HyperparameterSpace. """ - def __init__(self, *args, **kwds): - super().__init__(RecursiveDict.DEFAULT_SEPARATOR, *args, **kwds) + def __init__(self, *args, separator=None, **kwds): + super().__init__(*args, separator=separator, **kwds) class HyperparameterSpace(RecursiveDict): @@ -222,8 +223,8 @@ class HyperparameterSpace(RecursiveDict): Calling ``.rvs()`` on an ``HyperparameterSpace`` results in ``HyperparameterSamples``. """ - def __init__(self, *args, **kwds): - super().__init__(RecursiveDict.DEFAULT_SEPARATOR, *args, **kwds) + def __init__(self, *args, separator=None, **kwds): + super().__init__(*args, separator=separator, **kwds) def rvs(self) -> 'HyperparameterSamples': """ diff --git a/neuraxle/steps/flow.py b/neuraxle/steps/flow.py index 61d33f0c..1c4ddfac 100644 --- a/neuraxle/steps/flow.py +++ b/neuraxle/steps/flow.py @@ -295,12 +295,12 @@ def __init__(self, steps, hyperparams=None): if hyperparams is None: choices = list(self.keys())[:-1] - self.set_hyperparams(HyperparameterSamples({ + self.set_hyperparams({ CHOICE_HYPERPARAM: choices[0] - })) - self.set_hyperparams_space(HyperparameterSpace({ + }) + self.set_hyperparams_space({ CHOICE_HYPERPARAM: Choice(choices) - })) + }) def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, dict]): """ diff --git a/neuraxle/steps/loop.py b/neuraxle/steps/loop.py index 83c8e8a4..3094aa7c 100644 --- a/neuraxle/steps/loop.py +++ b/neuraxle/steps/loop.py @@ -180,7 +180,7 @@ def _copy_one_step_per_data_input(self, data_container): # One copy of step per data input: steps = [self.copy_op(self.wrapped).set_name('{}[{}]'.format(self.wrapped.name, i)) for i in range(len(data_container))] self.steps_as_tuple = [(step.name, step) for step in steps] - self.invalidate() + self._invalidate() def _fit_transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> ('BaseStep', DataContainer): fitted_steps_data_containers = [] diff --git a/testing/hyperparams/test_space.py b/testing/hyperparams/test_space.py index 87ad9de3..9b83cff1 100644 --- a/testing/hyperparams/test_space.py +++ b/testing/hyperparams/test_space.py @@ -24,8 +24,7 @@ import pytest from neuraxle.hyperparams.distributions import * -from neuraxle.hyperparams.space import flat_to_nested_dict, nested_dict_to_flat, HyperparameterSpace, \ - HyperparameterSamples +from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples, RecursiveDict hyperparams_flat_and_dict_pairs = [ # Pair 1: @@ -55,14 +54,14 @@ @pytest.mark.parametrize("flat,expected_dic", hyperparams_flat_and_dict_pairs) def test_flat_to_dict_hyperparams(flat: dict, expected_dic: dict): - dic = flat_to_nested_dict(flat) + dic = RecursiveDict(flat).to_nested_dict_as_dict_primitive() assert dict(dic) == dict(expected_dic) @pytest.mark.parametrize("expected_flat,dic", hyperparams_flat_and_dict_pairs) def test_dict_to_flat_hyperparams(expected_flat: dict, dic: dict): - flat = nested_dict_to_flat(dic) + flat = RecursiveDict(dic).to_flat_as_dict_primitive() pprint(dict(flat)) pprint(expected_flat) diff --git a/testing/test_optional.py b/testing/test_optional.py index 2b83787a..6bb4720f 100644 --- a/testing/test_optional.py +++ b/testing/test_optional.py @@ -7,8 +7,8 @@ def test_optional_should_disable_wrapped_step_when_disabled(): p = Optional(MultiplyByN(2), nullified_return_value=[]).set_hyperparams(HyperparameterSamples({ - 'enabled': False - })) + 'enabled': False + })) data_inputs = np.array(list(range(10))) outputs = p.transform(data_inputs) @@ -18,8 +18,8 @@ def test_optional_should_disable_wrapped_step_when_disabled(): def test_optional_should_enable_wrapped_step_when_enabled(): p = Optional(MultiplyByN(2), nullified_return_value=[]).set_hyperparams(HyperparameterSamples({ - 'enabled': True - })) + 'enabled': True + })) data_inputs = np.array(list(range(10))) outputs = p.transform(data_inputs) diff --git a/testing/test_pickle_checkpoint_step.py b/testing/test_pickle_checkpoint_step.py index c1504751..235e7748 100644 --- a/testing/test_pickle_checkpoint_step.py +++ b/testing/test_pickle_checkpoint_step.py @@ -91,7 +91,8 @@ def test_when_no_hyperparams_should_save_checkpoint_pickle(tmpdir: LocalPath): def test_when_hyperparams_should_save_checkpoint_pickle(tmpdir: LocalPath): tape = TapeCallbackFunction() pickle_checkpoint_step = DefaultCheckpoint() - pipeline = create_pipeline(tmpdir, pickle_checkpoint_step, tape, HyperparameterSamples({"a__learning_rate": 1})) + pipeline = create_pipeline(tmpdir, pickle_checkpoint_step, tape, + HyperparameterSamples({"a__learning_rate": 1})) pipeline, actual_data_inputs = pipeline.fit_transform(data_inputs, expected_outputs) diff --git a/testing/test_pipeline.py b/testing/test_pipeline.py index 7c66d119..c2d4433e 100644 --- a/testing/test_pipeline.py +++ b/testing/test_pipeline.py @@ -23,7 +23,7 @@ import pytest from neuraxle.hyperparams.distributions import RandInt, LogUniform -from neuraxle.hyperparams.space import nested_dict_to_flat, HyperparameterSpace +from neuraxle.hyperparams.space import HyperparameterSpace, RecursiveDict from neuraxle.pipeline import Pipeline from neuraxle.steps.misc import TransformCallbackStep, TapeCallbackFunction from neuraxle.steps.numpy import NumpyTranspose @@ -311,12 +311,12 @@ def test_pipeline_tosklearn(): z_ = a_["z"] assert z_.get_params()["learning_rate"] == 11 - p.set_params(**nested_dict_to_flat({ + p.set_params(**RecursiveDict({ "sk__b": { "a__z__learning_rate": 12, "b__learning_rate": 9 } - })) + }).to_flat()) # p.set_params(**{"sk__b__a__z__learning_rate": 12}) assert p.named_steps["sk"].p["b"].wrapped_sklearn_predictor.named_steps["a"]["z"].get_params()[ "learning_rate"] == 12 diff --git a/testing/test_recursive_dict.py b/testing/test_recursive_dict.py index 9de72f12..b3872c22 100644 --- a/testing/test_recursive_dict.py +++ b/testing/test_recursive_dict.py @@ -1,7 +1,34 @@ +import pytest + from neuraxle.hyperparams.space import RecursiveDict, HyperparameterSamples +POINT_SEPARATOR = '.' + -def test_recursive_dict_to_flat(): +@pytest.mark.parametrize("separator", ["__", ".", "___"]) +def test_recursive_dict_to_flat(separator): + dict_values = { + 'hp': 1, + 'stepa': { + 'hp': 2, + 'stepb': { + 'hp': 3 + } + } + } + r = RecursiveDict(separator=separator, **dict_values) + + r = r.to_flat() + + expected_dict_values = { + 'hp': 1, + 'stepa{}hp'.format(separator): 2, + 'stepa{0}stepb{0}hp'.format(separator): 3 + } + assert r == RecursiveDict(separator=separator, **expected_dict_values) + + +def test_recursive_dict_to_flat_different_separator(): dict_values = { 'hp': 1, 'stepa': { @@ -12,15 +39,17 @@ def test_recursive_dict_to_flat(): } } r = RecursiveDict(separator='__', **dict_values) + r['stepa'] = RecursiveDict(r['stepa'], separator='.') + r['stepa']['stepb'] = RecursiveDict(r['stepa']['stepb'], separator='$$$') r = r.to_flat() expected_dict_values = { 'hp': 1, - 'stepa__hp': 2, - 'stepa__stepb__hp': 3 + 'stepa.hp': 2, + 'stepa.stepb$$$hp': 3 } - assert r == RecursiveDict(separator='__', **expected_dict_values) + assert r.to_flat_as_dict_primitive() == expected_dict_values def test_recursive_dict_to_nested_dict(): @@ -85,11 +114,22 @@ def test_recursive_dict_copy_constructor(): 'stepa__hp': 2, 'stepa__stepb__hp': 3 } - r = RecursiveDict('__', RecursiveDict(**dict_values)) + r = RecursiveDict(RecursiveDict(**dict_values), separator='__') assert r == RecursiveDict(**dict_values) +def test_recursive_dict_copy_constructor_should_set_separator(): + dict_values = { + 'hp': 1, + 'stepa__hp': 2, + 'stepa__stepb__hp': 3 + } + r = RecursiveDict(RecursiveDict(**dict_values, separator=POINT_SEPARATOR)) + + assert r.separator == POINT_SEPARATOR + + def test_hyperparams_copy_constructor(): dict_values = { 'hp': 1, From d30f1aff9e516123a1fe3810c3e879b88cf76d0d Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 17:45:10 -0400 Subject: [PATCH 44/48] Change .toflat() return type for RecursiveDicT --- neuraxle/hyperparams/space.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index d2f34f06..a6aa309d 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -106,7 +106,7 @@ def __getitem__(self, item: str = None): return item_values - def to_flat(self) -> 'RecursiveDict': + def to_flat(self) -> dict: """ Will create an equivalent flat HyperparameterSamples. From 00379294f4dd53e22ea0201f0f962c72f4b14c52 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 17:49:42 -0400 Subject: [PATCH 45/48] Add back good docstring code example for set hyperparams space --- neuraxle/base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/neuraxle/base.py b/neuraxle/base.py index 2baed25f..f1e3c62b 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -959,6 +959,22 @@ def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpa """ Set step hyperparameters space. + Example : + + .. code-block:: python + + step.set_hyperparams_space(HyperparameterSpace({ + 'learning_rate': LogNormal(0.5, 0.5) + 'weight_decay': LogNormal(0.001, 0.0005) + })) + + step.update_hyperparams_space(HyperparameterSpace({ + 'learning_rate': LogNormal(0.5, 0.1) + })) + + assert step.get_hyperparams_space()['learning_rate'] == LogNormal(0.5, 0.1) + assert step.get_hyperparams_space()['weight_decay'] == LogNormal(0.001, 0.0005) + Example : .. code-block:: python From 808fa7f085366e5f3b70afa7ba8c8a56dacb0548 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 13 May 2020 18:12:57 -0400 Subject: [PATCH 46/48] Fix to_flat, and to_nested with different separators in RecursiveDict --- neuraxle/hyperparams/space.py | 32 +++++++++++++++++++++++++++----- neuraxle/steps/sklearn.py | 6 ++++-- testing/test_recursive_dict.py | 4 ++-- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index a6aa309d..f1c84057 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -161,13 +161,16 @@ def nested_dict_to_flat(self, dict_ctor): :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. :return: a flat hyperparameter dictionary. """ + if isinstance(dict_ctor, RecursiveDict): + ret = dict_ctor(self.separator) + else: + ret = dict_ctor() - ret = dict_ctor() for k, v in self.items(): if isinstance(v, dict) or isinstance(v, OrderedDict) or isinstance(v, dict_ctor): nested_dict = v if not isinstance(v, RecursiveDict): - nested_dict = RecursiveDict(v) + nested_dict = RecursiveDict(v, separator=self.separator) _ret = nested_dict.nested_dict_to_flat(dict_ctor=type(nested_dict)) for key, val in _ret.items(): ret[k + nested_dict.separator + key] = val @@ -182,8 +185,13 @@ def flat_to_nested_dict(self, dict_ctor=dict): :param dict_ctor: ``OrderedDict`` by default. Will use this as a class to create the new returned dict. :return: a nested hyperparameter dictionary. """ - pre_ret = dict_ctor() - ret = dict_ctor() + if isinstance(dict_ctor, RecursiveDict): + pre_ret = dict_ctor(self.separator) + ret = dict_ctor(self.separator) + else: + pre_ret = dict_ctor() + ret = dict_ctor() + for k, v in self.items(): k, _, key = k.partition(self.separator) if len(key) > 0: @@ -195,10 +203,24 @@ def flat_to_nested_dict(self, dict_ctor=dict): for k, v in pre_ret.items(): flat_dict = v if not isinstance(v, RecursiveDict): - flat_dict = RecursiveDict(v) + flat_dict = RecursiveDict(v, separator=self.separator) ret[k] = flat_dict.flat_to_nested_dict(dict_ctor=dict_ctor) return ret + def with_separator(self, separator): + """ + Create a new recursive dict that uses the given separator at each level. + + :param separator: + :return: + """ + return type(self)( + separator=separator, + **{ + key.replace(self.separator, separator): value if not isinstance(value, RecursiveDict) \ + else value.with_separator(separator) for key, value in self.items() + }) + class HyperparameterSamples(RecursiveDict): """ diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index 450a3eae..12a9fc90 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -31,7 +31,7 @@ from neuraxle.base import BaseStep, _HasChildrenMixin from neuraxle.hyperparams.distributions import LogUniform, Boolean -from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples +from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples, RecursiveDict from neuraxle.steps.numpy import NumpyTranspose from neuraxle.union import ModelStacking @@ -88,7 +88,9 @@ def _set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: # flatten the step hyperparams, and set the wrapped sklearn predictor params hyperparams = HyperparameterSamples(hyperparams) BaseStep._set_hyperparams(self, hyperparams.to_flat()) - self.wrapped_sklearn_predictor.set_params(**hyperparams.to_flat_as_dict_primitive()) + self.wrapped_sklearn_predictor.set_params( + **hyperparams.with_separator(RecursiveDict.DEFAULT_SEPARATOR).to_flat_as_dict_primitive() + ) return self diff --git a/testing/test_recursive_dict.py b/testing/test_recursive_dict.py index b3872c22..e25b132d 100644 --- a/testing/test_recursive_dict.py +++ b/testing/test_recursive_dict.py @@ -18,14 +18,14 @@ def test_recursive_dict_to_flat(separator): } r = RecursiveDict(separator=separator, **dict_values) - r = r.to_flat() + r = r.to_flat_as_dict_primitive() expected_dict_values = { 'hp': 1, 'stepa{}hp'.format(separator): 2, 'stepa{0}stepb{0}hp'.format(separator): 3 } - assert r == RecursiveDict(separator=separator, **expected_dict_values) + assert r == expected_dict_values def test_recursive_dict_to_flat_different_separator(): From f986af2687b75ded96ff50f38b76598775cc6b64 Mon Sep 17 00:00:00 2001 From: Guillaume Chevalier Date: Wed, 20 May 2020 22:30:53 -0400 Subject: [PATCH 47/48] Update neuraxle/base.py --- neuraxle/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index f1e3c62b..5cb57ac0 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -1778,7 +1778,7 @@ def _purge_apply_results(self, results) -> RecursiveDict: @abstractmethod def get_children(self) -> List[BaseStep]: """ - Get the list of all the children for that step. + Get the list of all the childs for that step. :return: """ From e44905fa33855315064b16946a8c5c47c8c2ad96 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Thu, 21 May 2020 14:29:45 -0400 Subject: [PATCH 48/48] All apply methods should Return A RecursiveDict --- neuraxle/base.py | 70 +++++++++++++++++----------------- neuraxle/hyperparams/space.py | 2 +- neuraxle/metaopt/random.py | 3 ++ neuraxle/metrics.py | 12 ++++-- neuraxle/steps/misc.py | 4 +- neuraxle/steps/sklearn.py | 2 +- testing/test_apply.py | 16 ++++---- testing/test_pipeline.py | 11 ++++-- testing/test_recursive_dict.py | 1 + testing/test_union.py | 2 +- 10 files changed, 68 insertions(+), 55 deletions(-) diff --git a/neuraxle/base.py b/neuraxle/base.py index 5cb57ac0..41f3b974 100644 --- a/neuraxle/base.py +++ b/neuraxle/base.py @@ -735,7 +735,7 @@ def teardown(self) -> 'BaseStep': self.is_initialized = False return self - def set_train(self, is_train: bool = True): + def set_train(self, is_train: bool = True) -> 'BaseStep': """ This method overrides the method of BaseStep to also consider the wrapped step as well as self. Set pipeline step mode to train or test. @@ -753,9 +753,9 @@ def set_train(self, is_train: bool = True): self.apply(method='_set_train', is_train=is_train) return self - def _set_train(self, is_train): + def _set_train(self, is_train) -> RecursiveDict: self.is_train = is_train - return self + return RecursiveDict() def set_name(self, name: str): """ @@ -831,11 +831,11 @@ def set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> 'B self.apply(method='_set_hyperparams', hyperparams=HyperparameterSamples(hyperparams).to_flat()) return self - def _set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]): + def _set_hyperparams(self, hyperparams: Union[HyperparameterSamples, Dict]) -> HyperparameterSamples: self._invalidate() hyperparams = HyperparameterSamples(hyperparams).to_flat() self.hyperparams = hyperparams if len(hyperparams) > 0 else self.hyperparams - return self + return self.hyperparams def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> 'BaseStep': """ @@ -873,8 +873,9 @@ def update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> self.apply(method='_update_hyperparams', hyperparams=HyperparameterSamples(hyperparams).to_flat()) return self - def _update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]): + def _update_hyperparams(self, hyperparams: Union[Dict, HyperparameterSamples]) -> HyperparameterSamples: self.hyperparams.update(HyperparameterSamples(hyperparams).to_flat()) + return self.hyperparams def get_hyperparams(self) -> HyperparameterSamples: """ @@ -894,7 +895,7 @@ def get_hyperparams(self) -> HyperparameterSamples: results: HyperparameterSamples = self.apply(method='_get_hyperparams') return results.to_flat() - def _get_hyperparams(self): + def _get_hyperparams(self) -> HyperparameterSamples: return HyperparameterSamples(self.hyperparams.to_flat_as_dict_primitive()) def set_params(self, **params) -> 'BaseStep': @@ -923,8 +924,9 @@ def set_params(self, **params) -> 'BaseStep': self.apply(method='_set_params', params=HyperparameterSamples(params).to_flat()) return self - def _set_params(self, params: dict): - return self.set_hyperparams(HyperparameterSamples(params)) + def _set_params(self, params: dict) -> HyperparameterSamples: + self.set_hyperparams(HyperparameterSamples(params)) + return self.hyperparams def get_params(self) -> HyperparameterSamples: """ @@ -952,8 +954,8 @@ def get_params(self) -> HyperparameterSamples: results: HyperparameterSamples = self.apply(method='_get_params') return results - def _get_params(self): - return self.get_hyperparams().to_flat_as_ordered_dict_primitive() + def _get_params(self) -> HyperparameterSamples: + return self.get_hyperparams().to_flat() def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> 'BaseStep': """ @@ -999,16 +1001,15 @@ def set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpa :func:`_HasChildrenMixin._apply`, :func:`_HasChildrenMixin._get_params` """ - self.apply(method='_set_hyperparams_space', hyperparams_space=HyperparameterSpace( - hyperparams_space).to_flat()) + self.apply(method='_set_hyperparams_space', hyperparams_space=HyperparameterSpace(hyperparams_space).to_flat()) return self - def _set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): + def _set_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> HyperparameterSpace: self._invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() self.hyperparams_space = HyperparameterSpace(hyperparams_space) if len( hyperparams_space) > 0 else self.hyperparams_space - return self + return self.hyperparams_space def update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> 'BaseStep': """ @@ -1044,11 +1045,11 @@ def update_hyperparams_space(self, hyperparams_space: Union[Dict, Hyperparameter hyperparams_space).to_flat()) return self - def _update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]): + def _update_hyperparams_space(self, hyperparams_space: Union[Dict, HyperparameterSpace]) -> HyperparameterSpace: self._invalidate() hyperparams_space = HyperparameterSamples(hyperparams_space).to_flat() self.hyperparams_space.update(HyperparameterSpace(hyperparams_space).to_flat()) - return self + return self.hyperparams_space def get_hyperparams_space(self) -> HyperparameterSpace: """ @@ -1072,7 +1073,7 @@ def get_hyperparams_space(self) -> HyperparameterSpace: results: HyperparameterSpace = self.apply(method='_get_hyperparams_space') return results.to_flat() - def _get_hyperparams_space(self): + def _get_hyperparams_space(self) -> HyperparameterSpace: return HyperparameterSpace(self.hyperparams_space.to_flat_as_dict_primitive()) def handle_inverse_transform(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: @@ -1119,16 +1120,26 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a kargs = ra.kargs + def _return_empty(*args, **kwargs): + return RecursiveDict() + + _method = _return_empty if isinstance(method, str) and hasattr(self, method) and callable(getattr(self, method)): - method = getattr(self, method) - else: + _method = getattr(self, method) + + if not isinstance(method, str): + _method = method kargs = [self] + list(kargs) try: - return method(*kargs, **ra.kwargs) + results = _method(*kargs, **ra.kwargs) + if not isinstance(results, RecursiveDict): + raise ValueError('Method {} must return a RecursiveDict because it is applied recursively.'.format(method)) + return results except Exception as err: print('{}: Failed to apply method {}.'.format(self.name, method)) print(traceback.format_stack()) + raise err def get_step_by_name(self, name): if self.name == name: @@ -1472,10 +1483,11 @@ def save(self, context: ExecutionContext, full_dump=False) -> 'BaseStep': def _initialize_if_needed(step): if not step.is_initialized: step.setup() - return step + return RecursiveDict() def _invalidate(step): step._invalidate() + return RecursiveDict() if full_dump: # initialize and invalidate steps to make sure that all steps will be saved @@ -1754,27 +1766,15 @@ def apply(self, method: Union[str, Callable], ra: _RecursiveArguments = None, *a def _apply_self(self, method: Union[str, Callable], ra: _RecursiveArguments): terminal_ra: _RecursiveArguments = ra[None] self_results: RecursiveDict = BaseStep.apply(self, method=method, ra=terminal_ra) - self_results: RecursiveDict = self._purge_apply_results(self_results) return self_results def _apply_childrens(self, results: RecursiveDict, method: Union[str, Callable], ra: _RecursiveArguments) -> RecursiveDict: for children in self.get_children(): children_results = children.apply(method=method, ra=ra[children.get_name()]) - - children_results = self._purge_apply_results(children_results) - results[children.get_name()] = children_results + results[children.get_name()] = RecursiveDict(children_results) return results - def _purge_apply_results(self, results) -> RecursiveDict: - if isinstance(results, RecursiveDict): - return results - - if isinstance(results, dict) and not isinstance(results, RecursiveDict): - return RecursiveDict(**results) - - return RecursiveDict() - @abstractmethod def get_children(self) -> List[BaseStep]: """ diff --git a/neuraxle/hyperparams/space.py b/neuraxle/hyperparams/space.py index f1c84057..243022bc 100644 --- a/neuraxle/hyperparams/space.py +++ b/neuraxle/hyperparams/space.py @@ -106,7 +106,7 @@ def __getitem__(self, item: str = None): return item_values - def to_flat(self) -> dict: + def to_flat(self) -> 'RecursiveDict': """ Will create an equivalent flat HyperparameterSamples. diff --git a/neuraxle/metaopt/random.py b/neuraxle/metaopt/random.py index a6378328..eaa34dc1 100644 --- a/neuraxle/metaopt/random.py +++ b/neuraxle/metaopt/random.py @@ -34,6 +34,7 @@ from neuraxle.base import MetaStepMixin, BaseStep, ExecutionContext, HandleOnlyMixin, ForceHandleOnlyMixin, \ EvaluableStepMixin from neuraxle.data_container import DataContainer +from neuraxle.hyperparams.space import RecursiveDict from neuraxle.steps.loop import StepClonerForEachDataInput from neuraxle.steps.numpy import NumpyConcatenateOuterBatch, NumpyConcatenateOnCustomAxis @@ -327,11 +328,13 @@ def disable_metrics(self): self.metrics_enabled = False if self.wrapped is not None: self.wrapped.apply('disable_metrics') + return RecursiveDict() def enable_metrics(self): self.metrics_enabled = True if self.wrapped is not None: self.wrapped.apply('enable_metrics') + return RecursiveDict() def _get_index_split(self, data_inputs): return math.floor(len(data_inputs) * (1 - self.test_size)) diff --git a/neuraxle/metrics.py b/neuraxle/metrics.py index 20035dac..45e3dfa5 100644 --- a/neuraxle/metrics.py +++ b/neuraxle/metrics.py @@ -23,6 +23,7 @@ from neuraxle.base import MetaStepMixin, BaseStep, ExecutionContext from neuraxle.data_container import DataContainer +from neuraxle.hyperparams.space import RecursiveDict class MetricsWrapper(MetaStepMixin, BaseStep): @@ -141,25 +142,26 @@ def _calculate_metrics_results(self, data_container: DataContainer): if self.print_metrics: self.print_fun(result) - def get_metrics(self) -> Dict: + def get_metrics(self) -> RecursiveDict: """ Get all metrics results using the transformed data container, and the metrics function dict. To be used with :func:`neuraxle.base.BaseStep.apply` method. :return: dict with the step name as key, and all of the training, and validation metrics as values """ - return { + return RecursiveDict({ 'train': self.metrics_results_train, 'validation': self.metrics_results_validation - } + }) - def toggle_metrics(self): + def toggle_metrics(self) -> RecursiveDict: """ Toggle metrics wrapper on and off to temporarily disable metrics if needed.. :return: """ self.enabled = not self.enabled + return RecursiveDict({self.name: self.enabled}) def disable_metrics(self): """ @@ -168,6 +170,7 @@ def disable_metrics(self): :return: """ self.enabled = False + return RecursiveDict({self.name: self.enabled}) def enable_metrics(self): """ @@ -176,3 +179,4 @@ def enable_metrics(self): :return: """ self.enabled = True + return RecursiveDict({self.name: self.enabled}) diff --git a/neuraxle/steps/misc.py b/neuraxle/steps/misc.py index 603699a2..38034c1d 100644 --- a/neuraxle/steps/misc.py +++ b/neuraxle/steps/misc.py @@ -27,6 +27,8 @@ import time from abc import ABC +from neuraxle.hyperparams.space import RecursiveDict + VALUE_CACHING = 'value_caching' from typing import List, Any @@ -185,7 +187,7 @@ def clear_callbacks(self): self.fit_callback_function.data = [] self.fit_callback_function.name_tape = [] - return cleared_callbacks + return RecursiveDict(cleared_callbacks) class CallbackWrapper(HandleOnlyMixin, MetaStepMixin, BaseStep): diff --git a/neuraxle/steps/sklearn.py b/neuraxle/steps/sklearn.py index 12a9fc90..6b9e9ac2 100644 --- a/neuraxle/steps/sklearn.py +++ b/neuraxle/steps/sklearn.py @@ -92,7 +92,7 @@ def _set_hyperparams(self, hyperparams: HyperparameterSamples) -> BaseStep: **hyperparams.with_separator(RecursiveDict.DEFAULT_SEPARATOR).to_flat_as_dict_primitive() ) - return self + return self.hyperparams.to_flat() def _get_hyperparams(self): if self.return_all_sklearn_default_params_on_get: diff --git a/testing/test_apply.py b/testing/test_apply.py index fb187a76..9203e98b 100644 --- a/testing/test_apply.py +++ b/testing/test_apply.py @@ -8,7 +8,7 @@ def test_apply_on_pipeline_with_positional_argument_should_call_method_on_each_steps(): pipeline = Pipeline([MultiplyByN(1), MultiplyByN(1)]) - pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({ + pipeline.apply('_set_hyperparams', hyperparams=HyperparameterSamples({ 'multiply_by': 2, 'MultiplyByN__multiply_by': 3, 'MultiplyByN1__multiply_by': 4 @@ -23,7 +23,7 @@ def test_apply_method_on_pipeline_should_call_method_on_each_steps(): pipeline = Pipeline([MultiplyByN(1), MultiplyByN(1)]) pipeline.apply( - lambda step: step.set_hyperparams(HyperparameterSamples({'multiply_by': 2})) + lambda step: step._set_hyperparams(HyperparameterSamples({'multiply_by': 2})) ) assert pipeline.get_hyperparams()['multiply_by'] == 2 @@ -34,7 +34,7 @@ def test_apply_method_on_pipeline_should_call_method_on_each_steps(): def test_apply_on_pipeline_with_meta_step_and_positional_argument(): pipeline = Pipeline([OutputTransformerWrapper(MultiplyByN(1)), MultiplyByN(1)]) - pipeline.apply('set_hyperparams', hyperparams=HyperparameterSamples({ + pipeline.apply('_set_hyperparams', hyperparams=HyperparameterSamples({ 'multiply_by': 2, 'OutputTransformerWrapper__multiply_by': 3, 'OutputTransformerWrapper__MultiplyByN__multiply_by': 4, @@ -51,7 +51,7 @@ def test_apply_method_on_pipeline_with_meta_step_should_call_method_on_each_step pipeline = Pipeline([OutputTransformerWrapper(MultiplyByN(1)), MultiplyByN(1)]) pipeline.apply( - lambda step: step.set_hyperparams(HyperparameterSamples({'multiply_by': 2})) + lambda step: step._set_hyperparams(HyperparameterSamples({'multiply_by': 2})) ) assert pipeline.get_hyperparams()['multiply_by'] == 2 @@ -70,7 +70,7 @@ def test_has_children_mixin_apply_should_apply_method_to_direct_childrends(): ]), ]) - p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ + p.apply('_set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ 'a__hp': 0, 'b__hp': 1, 'Pipeline__hp': 2 @@ -91,7 +91,7 @@ def test_has_children_mixin_apply_should_apply_method_to_recursive_childrends(): ]), ]) - p.apply('set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ + p.apply('_set_hyperparams', ra=None, hyperparams=HyperparameterSamples({ 'Pipeline__c__hp': 3, 'Pipeline__d__hp': 4 })) @@ -106,7 +106,7 @@ def test_has_children_mixin_apply_should_return_recursive_dict_to_direct_childre ('b', Identity().set_hyperparams(HyperparameterSamples({'hp': 1}))) ]) - results = p.apply('get_hyperparams', ra=None) + results = p.apply('_get_hyperparams', ra=None) assert results.to_flat_as_dict_primitive()['a__hp'] == 0 assert results.to_flat_as_dict_primitive()['b__hp'] == 1 @@ -120,7 +120,7 @@ def test_has_children_mixin_apply_should_return_recursive_dict_to_recursive_chil ]).set_hyperparams(HyperparameterSamples({'hp': 2})), ]) - results = p.apply('get_hyperparams', ra=None) + results = p.apply('_get_hyperparams', ra=None) results = results.to_flat_as_dict_primitive() assert results['Pipeline__hp'] == 2 diff --git a/testing/test_pipeline.py b/testing/test_pipeline.py index c2d4433e..b1f77c52 100644 --- a/testing/test_pipeline.py +++ b/testing/test_pipeline.py @@ -157,9 +157,9 @@ def test_pipeline_set_one_hyperparam_level_two_flat(): print(p.get_hyperparams()) assert p["b"]["a"].hyperparams["learning_rate"] == 7 - assert p["b"]["c"].hyperparams == dict() - assert p["b"].hyperparams == dict() - assert p["c"].hyperparams == dict() + assert p["b"]["c"].hyperparams.to_flat_as_dict_primitive() == dict() + assert p["b"].hyperparams.to_flat_as_dict_primitive() == {'a__learning_rate': 7} + assert p["c"].hyperparams.to_flat_as_dict_primitive() == dict() def test_pipeline_set_one_hyperparam_level_two_dict(): @@ -248,7 +248,10 @@ def test_pipeline_update_hyperparam_level_two_flat(): assert p["b"]["a"].hyperparams["learning_rate"] == 0.01 assert p["b"]["a"].hyperparams["other_hp"] == 8 assert p["b"]["c"].hyperparams == dict() - assert p["b"].hyperparams == dict() + assert p["b"].hyperparams.to_flat_as_dict_primitive() == { + 'a__learning_rate': 0.01, + 'a__other_hp': 8 + } assert p["c"].hyperparams == dict() diff --git a/testing/test_recursive_dict.py b/testing/test_recursive_dict.py index e25b132d..0458e219 100644 --- a/testing/test_recursive_dict.py +++ b/testing/test_recursive_dict.py @@ -1,5 +1,6 @@ import pytest +from neuraxle.base import Identity from neuraxle.hyperparams.space import RecursiveDict, HyperparameterSamples POINT_SEPARATOR = '.' diff --git a/testing/test_union.py b/testing/test_union.py index 2f173796..353fa8fb 100644 --- a/testing/test_union.py +++ b/testing/test_union.py @@ -78,7 +78,7 @@ def test_feature_union_should_apply_to_self_and_sub_steps(): ], joiner=NumpyTranspose()) ]) - p.apply(lambda step: step.set_hyperparams(HyperparameterSamples({'applied': True}))) + p.apply(lambda step: step._set_hyperparams(HyperparameterSamples({'applied': True}))) assert p.hyperparams['applied'] assert p['FeatureUnion'].hyperparams['applied']