Improving preprocessing

fedot/api/api_utils/api_data.py

@@ -1,4 +1,3 @@
-import sys
 from datetime import datetime
 from typing import Dict, Union
 from typing import Optional
@@ -34,14 +33,19 @@ def __init__(self, task: Task, use_input_preprocessing: bool = True):
         self.task = task
 
         self._recommendations = {}
-        self.preprocessor = DummyPreprocessor()
+
         if use_input_preprocessing:
             self.preprocessor = DataPreprocessor()
 
             # Dictionary with recommendations (e.g. 'cut' for cutting dataset, 'label_encoded'
             # to encode features using label encoder). Parameters for transformation provided also
-            self._recommendations = {'cut': self.preprocessor.cut_dataset,
-                                     'label_encoded': self.preprocessor.label_encoding_for_fit}
+            self._recommendations = {
+                'cut': self.preprocessor.cut_dataset,
+                'label_encoded': self.preprocessor.label_encoding_for_fit
+            }
+
+        else:
+            self.preprocessor = DummyPreprocessor()
 
         self.log = default_log(self)
 
@@ -133,18 +137,28 @@ def accept_and_apply_recommendations(self, input_data: Union[InputData, MultiMod
     def fit_transform(self, train_data: InputData) -> InputData:
         start_time = datetime.now()
         self.log.message('Preprocessing data')
-        memory_usage = convert_memory_size(sys.getsizeof(train_data.features))
+        memory_usage = convert_memory_size(train_data.memory_usage)
         features_shape = train_data.features.shape
         target_shape = train_data.target.shape
         self.log.message(
             f'Train Data (Original) Memory Usage: {memory_usage} Data Shapes: {features_shape, target_shape}')
 
+        self.log.debug('- Obligatory preprocessing started')
         train_data = self.preprocessor.obligatory_prepare_for_fit(data=train_data)
+
+        self.log.debug('- Optional preprocessing started')
         train_data = self.preprocessor.optional_prepare_for_fit(pipeline=Pipeline(), data=train_data)
+
+        self.log.debug('- Converting indexes for fitting started')
         train_data = self.preprocessor.convert_indexes_for_fit(pipeline=Pipeline(), data=train_data)
+
+        self.log.debug('- Reducing memory started')
+        train_data = self.preprocessor.reduce_memory_size(data=train_data)
+
         train_data.supplementary_data.is_auto_preprocessed = True
 
-        memory_usage = convert_memory_size(sys.getsizeof(train_data.features))
+        memory_usage = convert_memory_size(train_data.memory_usage)
+
         features_shape = train_data.features.shape
         target_shape = train_data.target.shape
         self.log.message(
@@ -156,7 +170,7 @@ def fit_transform(self, train_data: InputData) -> InputData:
     def transform(self, test_data: InputData, current_pipeline) -> InputData:
         start_time = datetime.now()
         self.log.message('Preprocessing data')
-        memory_usage = convert_memory_size(sys.getsizeof(test_data))
+        memory_usage = convert_memory_size(test_data.memory_usage)
         features_shape = test_data.features.shape
         target_shape = test_data.target.shape
         self.log.message(
@@ -168,7 +182,9 @@ def transform(self, test_data: InputData, current_pipeline) -> InputData:
         test_data = self.preprocessor.update_indices_for_time_series(test_data)
         test_data.supplementary_data.is_auto_preprocessed = True
 
-        memory_usage = convert_memory_size(sys.getsizeof(test_data))
+        test_data = self.preprocessor.reduce_memory_size(data=test_data)
+
+        memory_usage = convert_memory_size(test_data.memory_usage)
         features_shape = test_data.features.shape
         target_shape = test_data.target.shape
         self.log.message(

fedot/api/api_utils/predefined_model.py

@@ -8,26 +8,36 @@
 from fedot.core.pipelines.node import PipelineNode
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.verification import verify_pipeline
+from fedot.preprocessing.base_preprocessing import BasePreprocessor
 
 
 class PredefinedModel:
     def __init__(self, predefined_model: Union[str, Pipeline], data: InputData, log: LoggerAdapter,
-                 use_input_preprocessing: bool = True):
+                 use_input_preprocessing: bool = True, api_preprocessor: BasePreprocessor = None):
         self.predefined_model = predefined_model
         self.data = data
         self.log = log
-        self.pipeline = self._get_pipeline(use_input_preprocessing)
+        self.pipeline = self._get_pipeline(use_input_preprocessing, api_preprocessor)
 
-    def _get_pipeline(self, use_input_preprocessing: bool = True) -> Pipeline:
+    def _get_pipeline(self, use_input_preprocessing: bool = True,
+                      api_preprocessor: BasePreprocessor = None) -> Pipeline:
         if isinstance(self.predefined_model, Pipeline):
             pipelines = self.predefined_model
         elif self.predefined_model == 'auto':
             # Generate initial assumption automatically
             pipelines = AssumptionsBuilder.get(self.data).from_operations().build(
                 use_input_preprocessing=use_input_preprocessing)[0]
+
+            if use_input_preprocessing and api_preprocessor is not None:
+                pipelines.preprocessor = api_preprocessor
+
         elif isinstance(self.predefined_model, str):
             model = PipelineNode(self.predefined_model)
             pipelines = Pipeline(model, use_input_preprocessing=use_input_preprocessing)
+
+            if use_input_preprocessing and api_preprocessor is not None:
+                pipelines.preprocessor = api_preprocessor
+
         else:
             raise ValueError(f'{type(self.predefined_model)} is not supported as Fedot model')
 

fedot/api/main.py

@@ -176,9 +176,11 @@ def fit(self,
         with fedot_composer_timer.launch_fitting():
             if predefined_model is not None:
                 # Fit predefined model and return it without composing
-                self.current_pipeline = PredefinedModel(predefined_model, self.train_data, self.log,
-                                                        use_input_preprocessing=self.params.get(
-                                                            'use_input_preprocessing')).fit()
+                self.current_pipeline = PredefinedModel(
+                    predefined_model, self.train_data, self.log,
+                    use_input_preprocessing=self.params.get('use_input_preprocessing'),
+                    api_preprocessor=self.data_processor.preprocessor,
+                ).fit()
             else:
                 self.current_pipeline, self.best_models, self.history = self.api_composer.obtain_model(self.train_data)