diff --git a/docs/source/advanced/hyperparameters_tuning.rst b/docs/source/advanced/hyperparameters_tuning.rst index 1c39393db9..65484a2a50 100644 --- a/docs/source/advanced/hyperparameters_tuning.rst +++ b/docs/source/advanced/hyperparameters_tuning.rst @@ -1,8 +1,9 @@ + Tuning of Hyperparameters ========================= To tune pipeline hyperparameters you can use GOLEM. There are two ways: -1. Tuning of all models hyperparameters simultaneously. Implemented via ``SimultaneousTuner`` and ``IOptTuner`` classes. +1. Tuning of all models hyperparameters simultaneously. Implemented via ``SimultaneousTuner``, ``OptunaTuner`` and ``IOptTuner`` classes. 2. Tuning of models hyperparameters sequentially node by node optimizing metric value for the whole pipeline or tuning only one node hyperparametrs. Implemented via ``SequentialTuner`` class. @@ -16,22 +17,25 @@ using ``SimultaneousTuner`` is applied for composed pipeline and ``metric`` valu FEDOT uses tuners implementation from GOLEM, see `GOLEM documentation`_ for more information. .. list-table:: Tuners comparison - :widths: 10 30 30 30 + :widths: 10 30 30 30 30 :header-rows: 1 * - - ``SimultaneousTuner`` - ``SequentialTuner`` - ``IOptTuner`` + - ``OptunaTuner`` * - Based on - Hyperopt - Hyperopt - iOpt + - Optuna * - Type of tuning - Simultaneous - | Sequential or | for one node only - Simultaneous + - Simultaneous * - | Optimized | parameters - | categorical @@ -42,10 +46,14 @@ FEDOT uses tuners implementation from GOLEM, see `GOLEM documentation`_ for more | continuous - | discrete | continuous + - | categorical + | discrete + | continuous * - Algorithm type - stochastic - stochastic - deterministic + - stochastic * - | Supported | constraints - | timeout @@ -58,11 +66,22 @@ FEDOT uses tuners implementation from GOLEM, see `GOLEM documentation`_ for more | eval_time_constraint - | iterations | eval_time_constraint + - | timeout + | iterations + | early_stopping_rounds + | eval_time_constraint * - | Supports initial | point - Yes - No - No + - Yes + * - | Supports multi + | objective tuning + - No + - No + - No + - Yes Hyperopt based tuners usually take less time for one iteration, but ``IOptTuner`` is able to obtain much more stable results. @@ -488,7 +507,91 @@ Tuned pipeline structure: {'depth': 2, 'length': 3, 'nodes': [knnreg, knnreg, rfr]} knnreg - {'n_neighbors': 51} knnreg - {'n_neighbors': 40} - rfr - {'n_jobs': 1, 'max_features': 0.05324707031250003, 'min_samples_split': 12, 'min_samples_leaf': 11} + rfr - {'n_jobs': 1, 'max_features': 0.05324, 'min_samples_split': 12, 'min_samples_leaf': 11} + +Example for ``OptunaTuner``: + +.. code-block:: python + + from golem.core.tuning.optuna_tuner import OptunaTuner + from fedot.core.data.data import InputData + from fedot.core.pipelines.pipeline_builder import PipelineBuilder + from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder + from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum + from fedot.core.repository.tasks import TaskTypesEnum, Task + + task = Task(TaskTypesEnum.regression) + + tuner = OptunaTuner + + metric = RegressionMetricsEnum.MSE + + iterations = 100 + + train_data = InputData.from_csv('train_data.csv', task='regression') + + pipeline = PipelineBuilder().add_node('knnreg', branch_idx=0).add_branch('rfr', branch_idx=1) \ + .join_branches('knnreg').build() + + pipeline_tuner = TunerBuilder(task) \ + .with_tuner(tuner) \ + .with_metric(metric) \ + .with_iterations(iterations) \ + .build(train_data) + + tuned_pipeline = pipeline_tuner.tune(pipeline) + + tuned_pipeline.print_structure() + +Tuned pipeline structure: + +.. code-block:: python + + Pipeline structure: + {'depth': 2, 'length': 3, 'nodes': [knnreg, knnreg, rfr]} + knnreg - {'n_neighbors': 51} + knnreg - {'n_neighbors': 40} + rfr - {'n_jobs': 1, 'max_features': 0.05, 'min_samples_split': 12, 'min_samples_leaf': 11} + + +Multi objective tuning +^^^^^^^^^^^^^^^^^^^^^^ + +Multi objective tuning is available only for ``OptunaTuner``. Pass a list of metrics to ``.with_metric()`` +and obtain a list of tuned pipelines representing a pareto front after tuning. + +.. code-block:: python + + from typing import Iterable + from golem.core.tuning.optuna_tuner import OptunaTuner + from fedot.core.data.data import InputData + from fedot.core.pipelines.pipeline import Pipeline + from fedot.core.pipelines.pipeline_builder import PipelineBuilder + from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder + from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum + from fedot.core.repository.tasks import TaskTypesEnum, Task + + task = Task(TaskTypesEnum.regression) + + tuner = OptunaTuner + + metric = [RegressionMetricsEnum.MSE, RegressionMetricsEnum.MAE] + + iterations = 100 + + train_data = InputData.from_csv('train_data.csv', task='regression') + + pipeline = PipelineBuilder().add_node('knnreg', branch_idx=0).add_branch('rfr', branch_idx=1) \ + .join_branches('knnreg').build() + + pipeline_tuner = TunerBuilder(task) \ + .with_tuner(tuner) \ + .with_metric(metric) \ + .with_iterations(iterations) \ + .build(train_data) + + pareto_front: Iterable[Pipeline] = pipeline_tuner.tune(pipeline) + Sequential tuning ----------------- diff --git a/fedot/core/pipelines/tuning/tuner_builder.py b/fedot/core/pipelines/tuning/tuner_builder.py index 495c6e9b15..698de90a95 100644 --- a/fedot/core/pipelines/tuning/tuner_builder.py +++ b/fedot/core/pipelines/tuning/tuner_builder.py @@ -1,8 +1,10 @@ from datetime import timedelta -from typing import Type, Union +from typing import Type, Union, Iterable, Sequence +from golem.core.tuning.optuna_tuner import OptunaTuner from golem.core.tuning.simultaneous import SimultaneousTuner from golem.core.tuning.tuner_interface import BaseTuner +from golem.core.utilities.data_structures import ensure_wrapped_in_sequence from fedot.core.constants import DEFAULT_TUNING_ITERATIONS_NUMBER from fedot.core.data.data import InputData @@ -23,7 +25,7 @@ def __init__(self, task: Task): self.cv_folds = None self.validation_blocks = None self.n_jobs = -1 - self.metric: MetricsEnum = MetricByTask.get_default_quality_metrics(task.task_type)[0] + self.metric: Sequence[MetricsEnum] = MetricByTask.get_default_quality_metrics(task.task_type) self.iterations = DEFAULT_TUNING_ITERATIONS_NUMBER self.early_stopping_rounds = None self.timeout = timedelta(minutes=5) @@ -53,8 +55,8 @@ def with_n_jobs(self, n_jobs: int): self.n_jobs = n_jobs return self - def with_metric(self, metric: MetricType): - self.metric = metric + def with_metric(self, metrics: Union[MetricType, Iterable[MetricType]]): + self.metric = ensure_wrapped_in_sequence(metrics) return self def with_iterations(self, iterations: int): @@ -88,11 +90,16 @@ def with_adapter(self, adapter): return self def with_additional_params(self, **parameters): - self.additional_params = parameters + self.additional_params.update(parameters) return self def build(self, data: InputData) -> BaseTuner: - objective = MetricsObjective(self.metric) + if len(self.metric) > 1: + if self.tuner_class is OptunaTuner: + self.additional_params.update({'objectives_number': len(self.metric)}) + else: + raise ValueError('Multi objective tuning applicable only for OptunaTuner.') + objective = MetricsObjective(self.metric, is_multi_objective=len(self.metric) > 1) data_splitter = DataSourceSplitter(self.cv_folds, validation_blocks=self.validation_blocks) data_producer = data_splitter.build(data) objective_evaluate = PipelineObjectiveEvaluate(objective, data_producer, diff --git a/test/integration/pipelines/tuning/test_pipeline_tuning.py b/test/integration/pipelines/tuning/test_pipeline_tuning.py index fd08789dc9..34cdb062ab 100644 --- a/test/integration/pipelines/tuning/test_pipeline_tuning.py +++ b/test/integration/pipelines/tuning/test_pipeline_tuning.py @@ -4,13 +4,16 @@ import pytest from golem.core.tuning.hyperopt_tuner import get_node_parameters_for_hyperopt from golem.core.tuning.iopt_tuner import IOptTuner +from golem.core.tuning.optuna_tuner import OptunaTuner from golem.core.tuning.sequential import SequentialTuner from golem.core.tuning.simultaneous import SimultaneousTuner +from golem.core.utilities.data_structures import ensure_wrapped_in_sequence from hyperopt import hp from hyperopt.pyll.stochastic import sample as hp_sample -from sklearn.metrics import mean_squared_error as mse, accuracy_score as acc -from fedot.core.data.data import InputData, OutputData +from examples.simple.time_series_forecasting.ts_pipelines import ts_complex_ridge_smoothing_pipeline, \ + ts_glm_pipeline +from fedot.core.data.data import InputData from fedot.core.data.data_split import train_test_data_setup from fedot.core.operations.evaluation.operation_implementations.models.ts_implementations.statsmodels import \ GLMImplementation @@ -21,6 +24,7 @@ from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum, ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import fedot_project_root, NESTED_PARAMS_LABEL +from test.unit.multimodal.data_generators import get_single_task_multimodal_tabular_data, get_multimodal_pipeline from test.unit.tasks.test_forecasting import get_ts_data @@ -52,6 +56,18 @@ def multi_classification_dataset(): return InputData.from_csv(os.path.join(test_file_path, file), task=Task(TaskTypesEnum.classification)) +@pytest.fixture() +def ts_forecasting_dataset(): + train_data, _ = get_ts_data(n_steps=700, forecast_length=20) + return train_data + + +@pytest.fixture() +def multimodal_dataset(): + data, _ = get_single_task_multimodal_tabular_data() + return data + + def get_simple_regr_pipeline(operation_type='rfr'): final = PipelineNode(operation_type=operation_type) pipeline = Pipeline(final) @@ -109,6 +125,15 @@ def get_class_pipelines(): return simple_pipelines + [get_complex_class_pipeline()] +def get_ts_forecasting_pipelines(): + pipelines = [ts_glm_pipeline(), ts_complex_ridge_smoothing_pipeline()] + return pipelines + + +def get_multimodal_pipelines(): + return [get_multimodal_pipeline()] + + def get_regr_operation_types(): return ['lgbmreg'] @@ -118,7 +143,7 @@ def get_class_operation_types(): def get_regr_losses(): - return [RegressionMetricsEnum.RMSE] + return [RegressionMetricsEnum.RMSE, RegressionMetricsEnum.MAPE] def get_class_losses(): @@ -183,24 +208,14 @@ def get_not_default_search_space(): return PipelineSearchSpace(custom_search_space=custom_search_space) -def custom_maximized_metrics(real_data: InputData, pred_data: OutputData): - mse_value = mse(real_data.target, pred_data.predict, squared=False) - return -(mse_value + 2) * 0.5 - - -def custom_minimized_metrics(real_data: InputData, pred_data: OutputData): - acc_value = acc(real_data.target, pred_data.predict) - return 100 - (acc_value + 2) * 0.5 - - def run_pipeline_tuner(train_data, pipeline, loss_function, tuner=SimultaneousTuner, search_space=PipelineSearchSpace(), cv=None, - iterations=1, - early_stopping_rounds=None): + iterations=3, + early_stopping_rounds=None, **kwargs): # Pipeline tuning pipeline_tuner = TunerBuilder(train_data.task) \ .with_tuner(tuner) \ @@ -209,6 +224,7 @@ def run_pipeline_tuner(train_data, .with_iterations(iterations) \ .with_early_stopping_rounds(early_stopping_rounds) \ .with_search_space(search_space) \ + .with_additional_params(**kwargs) \ .build(train_data) tuned_pipeline = pipeline_tuner.tune(pipeline) return pipeline_tuner, tuned_pipeline @@ -220,7 +236,7 @@ def run_node_tuner(train_data, search_space=PipelineSearchSpace(), cv=None, node_index=0, - iterations=1, + iterations=3, early_stopping_rounds=None): # Pipeline tuning node_tuner = TunerBuilder(train_data.task) \ @@ -252,8 +268,10 @@ def test_custom_params_setter(data_fixture, request): @pytest.mark.parametrize('data_fixture, pipelines, loss_functions', [('regression_dataset', get_regr_pipelines(), get_regr_losses()), ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses())]) -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner]) + ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), + ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), + ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) +@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) def test_pipeline_tuner_correct(data_fixture, pipelines, loss_functions, request, tuner): """ Test all tuners for pipeline """ data = request.getfixturevalue(data_fixture) @@ -268,14 +286,11 @@ def test_pipeline_tuner_correct(data_fixture, pipelines, loss_functions, request loss_function=loss_function, cv=cv) assert pipeline_tuner.obtained_metric is not None + assert tuned_pipeline is not None assert not tuned_pipeline.is_fitted - is_tuning_finished = True - - assert is_tuning_finished - -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner]) +@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) def test_pipeline_tuner_with_no_parameters_to_tune(classification_dataset, tuner): pipeline = get_pipeline_with_no_params_to_tune() pipeline_tuner, tuned_pipeline = run_pipeline_tuner(tuner=tuner, @@ -284,11 +299,12 @@ def test_pipeline_tuner_with_no_parameters_to_tune(classification_dataset, tuner loss_function=ClassificationMetricsEnum.ROCAUC, iterations=20) assert pipeline_tuner.obtained_metric is not None + assert tuned_pipeline is not None assert pipeline_tuner.obtained_metric == pipeline_tuner.init_metric assert not tuned_pipeline.is_fitted -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner]) +@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) def test_pipeline_tuner_with_initial_params(classification_dataset, tuner): """ Test all tuners for pipeline with initial parameters """ # a model @@ -302,14 +318,17 @@ def test_pipeline_tuner_with_initial_params(classification_dataset, tuner): loss_function=ClassificationMetricsEnum.ROCAUC, iterations=20) assert pipeline_tuner.obtained_metric is not None + assert tuned_pipeline is not None assert not tuned_pipeline.is_fitted @pytest.mark.parametrize('data_fixture, pipelines, loss_functions', [('regression_dataset', get_regr_pipelines(), get_regr_losses()), ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses())]) -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner]) + ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), + ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), + ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) +@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) def test_pipeline_tuner_with_custom_search_space(data_fixture, pipelines, loss_functions, request, tuner): """ Test tuners with different search spaces """ data = request.getfixturevalue(data_fixture) @@ -317,22 +336,21 @@ def test_pipeline_tuner_with_custom_search_space(data_fixture, pipelines, loss_f search_spaces = [PipelineSearchSpace(), get_not_default_search_space()] for search_space in search_spaces: - pipeline_tuner, _ = run_pipeline_tuner(tuner=tuner, - train_data=train_data, - pipeline=pipelines[0], - loss_function=loss_functions[0], - search_space=search_space) + pipeline_tuner, tuned_pipeline = run_pipeline_tuner(tuner=tuner, + train_data=train_data, + pipeline=pipelines[0], + loss_function=loss_functions[0], + search_space=search_space) assert pipeline_tuner.obtained_metric is not None - - is_tuning_finished = True - - assert is_tuning_finished + assert tuned_pipeline is not None @pytest.mark.parametrize('data_fixture, pipelines, loss_functions', [('regression_dataset', get_regr_pipelines(), get_regr_losses()), ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses())]) + ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), + ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), + ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) def test_certain_node_tuning_correct(data_fixture, pipelines, loss_functions, request): """ Test SequentialTuner for particular node based on hyperopt library """ data = request.getfixturevalue(data_fixture) @@ -347,16 +365,15 @@ def test_certain_node_tuning_correct(data_fixture, pipelines, loss_functions, re cv=cv) assert node_tuner.obtained_metric is not None assert not tuned_pipeline.is_fitted - - is_tuning_finished = True - - assert is_tuning_finished + assert tuned_pipeline is not None @pytest.mark.parametrize('data_fixture, pipelines, loss_functions', [('regression_dataset', get_regr_pipelines(), get_regr_losses()), ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses())]) + ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), + ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), + ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) def test_certain_node_tuner_with_custom_search_space(data_fixture, pipelines, loss_functions, request): """ Test SequentialTuner for particular node with different search spaces """ data = request.getfixturevalue(data_fixture) @@ -364,19 +381,16 @@ def test_certain_node_tuner_with_custom_search_space(data_fixture, pipelines, lo search_spaces = [PipelineSearchSpace(), get_not_default_search_space()] for search_space in search_spaces: - node_tuner, _ = run_node_tuner(train_data=train_data, - pipeline=pipelines[0], - loss_function=loss_functions[0], - search_space=search_space) + node_tuner, tuned_pipeline = run_node_tuner(train_data=train_data, + pipeline=pipelines[0], + loss_function=loss_functions[0], + search_space=search_space) assert node_tuner.obtained_metric is not None - - is_tuning_finished = True - - assert is_tuning_finished + assert tuned_pipeline is not None @pytest.mark.parametrize('n_steps', [100, 133, 217, 300]) -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner]) +@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) def test_ts_pipeline_with_stats_model(n_steps, tuner): """ Tests tuners for time series forecasting task with AR model """ train_data, test_data = get_ts_data(n_steps=n_steps, forecast_length=5) @@ -390,11 +404,9 @@ def test_ts_pipeline_with_stats_model(n_steps, tuner): .with_metric(RegressionMetricsEnum.MSE) \ .with_iterations(3) \ .with_search_space(search_space).build(train_data) - _ = tuner_ar.tune(ar_pipeline) - - is_tuning_finished = True - - assert is_tuning_finished + tuned_pipeline = tuner_ar.tune(ar_pipeline) + assert tuned_pipeline is not None + assert tuner_ar.obtained_metric is not None @pytest.mark.parametrize('data_fixture', ['tiny_classification_dataset']) @@ -489,7 +501,7 @@ def test_complex_search_space(): assert params['link'] in GLMImplementation.family_distribution[params['family']]['available_links'] -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner]) +@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) def test_complex_search_space_tuning_correct(tuner): """ Tests SimultaneousTuner for time series forecasting task with GLM model that has a complex glm search space""" train_data, test_data = get_ts_data(n_steps=700, forecast_length=20) @@ -508,3 +520,30 @@ def test_complex_search_space_tuning_correct(tuner): assert initial_parameters == found_parameters else: assert initial_parameters != found_parameters + + +@pytest.mark.parametrize('data_fixture, pipelines, loss_functions', + [('regression_dataset', get_regr_pipelines(), get_regr_losses()), + ('classification_dataset', get_class_pipelines(), get_class_losses()), + ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), + ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), + ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) +@pytest.mark.parametrize('tuner', [OptunaTuner]) +def test_multiobj_tuning(data_fixture, pipelines, loss_functions, request, tuner): + """ Test multi objective tuning is correct """ + data = request.getfixturevalue(data_fixture) + cvs = [None, 2] + + for pipeline in pipelines: + for cv in cvs: + pipeline_tuner, tuned_pipelines = run_pipeline_tuner(tuner=tuner, + train_data=data, + pipeline=pipeline, + loss_function=loss_functions, + cv=cv, + iterations=10) + assert tuned_pipelines is not None + assert all([tuned_pipeline is not None for tuned_pipeline in ensure_wrapped_in_sequence(tuned_pipelines)]) + for metrics in pipeline_tuner.obtained_metric: + assert len(metrics) == len(loss_functions) + assert all(metric is not None for metric in metrics)