Fix some issues (#230)

Fix Problem with OptHistory #229
Fix Refactor utilities #173
Fix potential problem with n_jobs. It may appear if some multiprocessing modules cannot work with n_jobs < 0. Now n_jobs is checked and converted to positive value in GraphRequirements
Fix problem with retaining composing time calculation. Initial generation is taked as full generation, but it does not consume any time for preparing. Also fix problem with generation num increment.
Change version info from `0.3.3` to `0.4.0`.

docs/source/conf.py

@@ -24,7 +24,7 @@
 author = 'NSS Lab'
 
 # The full version, including alpha/beta/rc tags
-release = '0.3.3'
+release = '0.4.0'
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be

examples/synthetic_graph_evolution/utils.py

@@ -10,7 +10,7 @@
 
 from golem.core.adapter.nx_adapter import BaseNetworkxAdapter
 from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 from golem.visualisation.graph_viz import GraphVisualizer
 
 

golem/core/adapter/adapt_registry.py

@@ -2,7 +2,7 @@
 from functools import partial
 from typing import Callable
 
-from golem.core.utilities.singleton_meta import SingletonMeta
+from golem.utilities.singleton_meta import SingletonMeta
 
 
 class AdaptRegistry(metaclass=SingletonMeta):

golem/core/dag/graph_utils.py

@@ -1,6 +1,6 @@
 from typing import Sequence, List, TYPE_CHECKING, Callable, Union
 
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 if TYPE_CHECKING:
     from golem.core.dag.graph import Graph

golem/core/dag/linked_graph.py

@@ -8,7 +8,7 @@
 from golem.core.dag.graph_node import GraphNode
 from golem.core.dag.graph_utils import ordered_subnodes_hierarchy, node_depth, graph_has_cycle
 from golem.core.paths import copy_doc
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence, Copyable, remove_items
+from golem.utilities.data_structures import ensure_wrapped_in_sequence, Copyable, remove_items
 
 NodePostprocessCallable = Callable[[Graph, Sequence[GraphNode]], Any]
 

golem/core/dag/linked_graph_node.py

@@ -1,6 +1,6 @@
 from typing import Union, Optional, Iterable, List
 from golem.core.dag.graph_node import GraphNode
-from golem.core.utilities.data_structures import UniqueList
+from golem.utilities.data_structures import UniqueList
 
 
 class LinkedGraphNode(GraphNode):

golem/core/log.py

@@ -8,7 +8,7 @@
 from logging.handlers import RotatingFileHandler
 from typing import Optional, Tuple, Union
 
-from golem.core.utilities.singleton_meta import SingletonMeta
+from golem.utilities.singleton_meta import SingletonMeta
 from golem.core.paths import default_data_dir
 
 DEFAULT_LOG_PATH = pathlib.Path(default_data_dir(), 'log.log')

golem/core/optimisers/adaptive/agent_trainer.py

@@ -16,7 +16,7 @@
 from golem.core.optimisers.opt_history_objects.individual import Individual
 from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
 from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator
-from golem.core.utilities.data_structures import unzip
+from golem.utilities.data_structures import unzip
 
 
 class AgentTrainer:

golem/core/optimisers/advisor.py

@@ -1,6 +1,6 @@
 from typing import List, Any, TypeVar, Generic
 
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 NodeType = TypeVar('NodeType')
 

golem/core/optimisers/fitness/fitness.py

@@ -3,7 +3,7 @@
 
 import numpy as np
 
-from golem.core.utilities.data_structures import Comparable
+from golem.utilities.data_structures import Comparable
 
 
 class Fitness(Comparable):

golem/core/optimisers/genetic/evaluation.py

@@ -7,7 +7,7 @@
 from functools import partial
 from typing import List, Optional, Sequence, Tuple, TypeVar, Dict
 
-from joblib import Parallel, cpu_count, delayed
+from joblib import Parallel, delayed
 
 from golem.core.adapter import BaseOptimizationAdapter
 from golem.core.dag.graph import Graph
@@ -18,8 +18,9 @@
 from golem.core.optimisers.objective import GraphFunction, ObjectiveFunction
 from golem.core.optimisers.opt_history_objects.individual import GraphEvalResult
 from golem.core.optimisers.timer import Timer, get_forever_timer
-from golem.core.utilities.serializable import Serializable
+from golem.utilities.serializable import Serializable
 from golem.utilities.memory import MemoryAnalytics
+from golem.utilities.utilities import determine_n_jobs
 
 # the percentage of successful evaluations,
 # at which evolution is not threatened with stagnation at the moment
@@ -276,17 +277,3 @@ def evaluate_population(self, individuals: PopulationT) -> PopulationT:
         individuals_evaluated = self.apply_evaluation_results(individuals_to_evaluate, evaluation_results)
         evaluated_population = individuals_evaluated + individuals_to_skip
         return evaluated_population
-
-
-def determine_n_jobs(n_jobs=-1, logger=None):
-    cpu_num = cpu_count()
-    if n_jobs > cpu_num:
-        n_jobs = cpu_num
-    elif n_jobs <= 0:
-        if n_jobs <= -cpu_num - 1 or n_jobs == 0:
-            raise ValueError(f"Unproper `n_jobs` = {n_jobs}. "
-                             f"`n_jobs` should be between ({-cpu_num}, {cpu_num}) except 0")
-        n_jobs = cpu_num + 1 + n_jobs
-    if logger:
-        logger.info(f"Number of used CPU's: {n_jobs}")
-    return n_jobs

golem/core/optimisers/genetic/operators/base_mutations.py

@@ -11,7 +11,7 @@
 from golem.core.optimisers.opt_node_factory import OptNodeFactory
 from golem.core.optimisers.optimization_parameters import GraphRequirements
 from golem.core.optimisers.optimizer import GraphGenerationParams, AlgorithmParameters
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 if TYPE_CHECKING:
     from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters

golem/core/optimisers/genetic/operators/crossover.py

@@ -13,7 +13,7 @@
 from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator
 from golem.core.optimisers.optimization_parameters import GraphRequirements
 from golem.core.optimisers.optimizer import GraphGenerationParams
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 if TYPE_CHECKING:
     from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters

golem/core/optimisers/genetic/operators/elitism.py

@@ -1,7 +1,7 @@
 from random import shuffle
 
 from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 
 class ElitismTypesEnum(Enum):

golem/core/optimisers/genetic/operators/inheritance.py

@@ -2,7 +2,7 @@
 
 from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator
 from golem.core.optimisers.genetic.operators.selection import Selection
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 if TYPE_CHECKING:
     from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters

golem/core/optimisers/genetic/operators/regularization.py

@@ -7,7 +7,7 @@
 from golem.core.optimisers.opt_history_objects.individual import Individual
 from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator
 from golem.core.optimisers.optimizer import GraphGenerationParams
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 if TYPE_CHECKING:
     from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters

golem/core/optimisers/genetic/operators/reproduction.py

@@ -10,7 +10,7 @@
 from golem.core.optimisers.genetic.operators.operator import PopulationT, EvaluationOperator
 from golem.core.optimisers.genetic.operators.selection import Selection
 from golem.core.optimisers.populational_optimizer import EvaluationAttemptsError
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 
 class ReproductionController:

golem/core/optimisers/genetic/operators/selection.py

@@ -5,7 +5,7 @@
 from typing import Callable, List, Optional
 
 from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 
 class SelectionTypesEnum(Enum):

golem/core/optimisers/genetic/parameters/population_size.py

@@ -7,8 +7,8 @@
 from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum
 from golem.core.optimisers.genetic.operators.operator import PopulationT
 from golem.core.optimisers.genetic.parameters.parameter import AdaptiveParameter
-from golem.core.utilities.data_structures import BidirectionalIterator
-from golem.core.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator
+from golem.utilities.data_structures import BidirectionalIterator
+from golem.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator
 
 PopulationSize = AdaptiveParameter[int]
 

golem/core/optimisers/opt_history_objects/generation.py

@@ -4,7 +4,7 @@
 from copy import deepcopy, copy
 from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union
 
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 if TYPE_CHECKING:
     from golem.core.optimisers.opt_history_objects.individual import Individual

golem/core/optimisers/opt_history_objects/opt_history.py

@@ -99,7 +99,7 @@ def save_current_results(self, save_dir: Optional[os.PathLike] = None):
             last_gen = self.generations[last_gen_id]
             for individual in last_gen:
                 ind_path = Path(save_dir, str(last_gen_id), str(individual.uid))
-                ind_path.mkdir(exist_ok=True)
+                ind_path.mkdir(exist_ok=True, parents=True)
                 individual.save(json_file_path=ind_path / f'{str(individual.uid)}.json')
         except Exception as ex:
             self._log.exception(ex)

golem/core/optimisers/opt_history_objects/parent_operator.py

@@ -5,7 +5,7 @@
 from uuid import uuid4
 
 from golem.core.optimisers.opt_history_objects.individual import Individual
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 
 @dataclass(frozen=True)

golem/core/optimisers/optimization_parameters.py

@@ -5,6 +5,7 @@
 from typing import Optional
 
 from golem.core.paths import default_data_dir
+from golem.utilities.utilities import determine_n_jobs
 
 
 @dataclass
@@ -85,9 +86,9 @@ class GraphRequirements(OptimizationParameters):
     max_arity: int = 4
 
     def __post_init__(self):
-        excluded_fields = ['n_jobs']
+        # check and convert n_jobs to non-negative
+        self.n_jobs = determine_n_jobs(self.n_jobs)
+
         for field_name, field_value in dataclasses.asdict(self).items():
-            if field_name in excluded_fields:
-                continue
             if isinstance(field_value, Number) and field_value < 0:
                 raise ValueError(f'Value of {field_name} must be non-negative')

golem/core/optimisers/optimizer.py

@@ -18,7 +18,7 @@
 from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
 from golem.core.optimisers.opt_node_factory import DefaultOptNodeFactory, OptNodeFactory
 from golem.core.optimisers.random_graph_factory import RandomGraphFactory, RandomGrowthGraphFactory
-from golem.core.utilities.random import RandomStateHandler
+from golem.utilities.random import RandomStateHandler
 
 STRUCTURAL_DIVERSITY_FREQUENCY_CHECK = 5
 

golem/core/optimisers/populational_optimizer.py

@@ -13,7 +13,7 @@
 from golem.core.optimisers.optimization_parameters import GraphRequirements
 from golem.core.optimisers.optimizer import GraphGenerationParams, GraphOptimizer, AlgorithmParameters
 from golem.core.optimisers.timer import OptimisationTimer
-from golem.core.utilities.grouped_condition import GroupedCondition
+from golem.utilities.grouped_condition import GroupedCondition
 
 
 class PopulationalOptimizer(GraphOptimizer):
@@ -57,7 +57,7 @@ def __init__(self,
         max_stagnation_time = requirements.early_stopping_timeout or self.timer.timeout
         self.stop_optimization = \
             GroupedCondition(results_as_message=True).add_condition(
-                lambda: self.timer.is_time_limit_reached(self.current_generation_num),
+                lambda: self.timer.is_time_limit_reached(self.current_generation_num - 1),
                 'Optimisation stopped: Time limit is reached'
             ).add_condition(
                 lambda: (requirements.num_of_generations is not None and

golem/core/optimisers/random/random_mutation_optimizer.py

@@ -10,7 +10,7 @@
 from golem.core.optimisers.optimizer import GraphGenerationParams
 from golem.core.optimisers.populational_optimizer import PopulationalOptimizer
 from golem.core.optimisers.random.random_search import RandomSearchOptimizer
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 
 class PopulationalRandomMutationOptimizer(PopulationalOptimizer):

golem/core/optimisers/random/random_search.py

@@ -12,7 +12,7 @@
 from golem.core.optimisers.optimization_parameters import GraphRequirements
 from golem.core.optimisers.optimizer import GraphOptimizer, GraphGenerationParams
 from golem.core.optimisers.timer import OptimisationTimer
-from golem.core.utilities.grouped_condition import GroupedCondition
+from golem.utilities.grouped_condition import GroupedCondition
 
 
 class RandomSearchOptimizer(GraphOptimizer):

golem/core/optimisers/timer.py

@@ -48,9 +48,9 @@ def __init__(self, timeout: datetime.timedelta = None):
 
     def _is_next_iteration_possible(self, time_constraint: float, iteration_num: int = None) -> bool:
         minutes = self.minutes_from_start
-        if iteration_num is not None:
+        if iteration_num is not None and iteration_num != 0:
             evo_proc_minutes = minutes - self.init_time
-            possible = time_constraint > (minutes + (evo_proc_minutes / (iteration_num + 1)))
+            possible = time_constraint > (minutes + (evo_proc_minutes / iteration_num))
         else:
             possible = time_constraint > minutes
         if not possible:

golem/core/tuning/tuner_interface.py

@@ -14,7 +14,7 @@
 from golem.core.optimisers.graph import OptGraph
 from golem.core.optimisers.objective import ObjectiveEvaluate, ObjectiveFunction
 from golem.core.tuning.search_space import SearchSpace, convert_parameters
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 DomainGraphForTune = TypeVar('DomainGraphForTune')
 

golem/serializers/serializer.py

@@ -88,7 +88,7 @@ def _register_default_coders():
         from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator
         from golem.core.optimisers.fitness.fitness import Fitness
         from golem.core.optimisers.objective.objective import ObjectiveInfo
-        from golem.core.utilities.data_structures import ComparableEnum
+        from golem.utilities.data_structures import ComparableEnum
 
         from .any_serialization import any_from_json, any_to_json
 

golem/structural_analysis/graph_sa/results/sa_analysis_results.py

@@ -6,7 +6,7 @@
 from golem.core.dag.graph import Graph
 from golem.core.log import default_log
 from golem.core.paths import project_root
-from golem.core.utilities.serializable import Serializable
+from golem.utilities.serializable import Serializable
 from golem.serializers import Serializer
 from golem.structural_analysis.graph_sa.results.deletion_sa_approach_result import DeletionSAApproachResult
 from golem.structural_analysis.graph_sa.results.object_sa_result import ObjectSAResult, \

golem/core/utilities/sequence_iterator.py → golem/utilities/sequence_iterator.py

@@ -1,6 +1,6 @@
 from typing import Callable, Optional
 
-from golem.core.utilities.data_structures import BidirectionalIterator
+from golem.utilities.data_structures import BidirectionalIterator
 
 
 class SequenceIterator(BidirectionalIterator[int]):

golem/utilities/utilities.py

@@ -0,0 +1,15 @@
+from joblib import cpu_count
+
+
+def determine_n_jobs(n_jobs=-1, logger=None):
+    cpu_num = cpu_count()
+    if n_jobs > cpu_num:
+        n_jobs = cpu_num
+    elif n_jobs <= 0:
+        if n_jobs <= -cpu_num - 1 or n_jobs == 0:
+            raise ValueError(f"Unproper `n_jobs` = {n_jobs}. "
+                             f"`n_jobs` should be between ({-cpu_num}, {cpu_num}) except 0")
+        n_jobs = cpu_num + 1 + n_jobs
+    if logger:
+        logger.info(f"Number of used CPU's: {n_jobs}")
+    return n_jobs

golem/visualisation/opt_history/multiple_fitness_line.py

@@ -8,7 +8,7 @@
 
 from golem.core.log import default_log
 from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 from golem.visualisation.opt_history.arg_constraint_wrapper import ArgConstraintWrapper
 from golem.visualisation.opt_history.fitness_line import setup_fitness_plot
 from golem.visualisation.opt_history.utils import show_or_save_figure

setup.py

@@ -8,7 +8,7 @@
 
 # The text of the README file
 NAME = 'thegolem'
-VERSION = '0.3.3'
+VERSION = '0.4.0'
 AUTHOR = 'NSS Lab'
 SHORT_DESCRIPTION = 'Framework for Graph Optimization and Learning by Evolutionary Methods'
 

test/unit/optimizers/gp_operators/test_population_size.py

@@ -1,13 +1,11 @@
-import pytest
-
 from golem.core.optimisers.archive import GenerationKeeper
 from golem.core.optimisers.fitness import SingleObjFitness
-from golem.core.optimisers.genetic.parameters.population_size import PopulationSize, AdaptivePopulationSize, \
+from golem.core.optimisers.genetic.parameters.population_size import AdaptivePopulationSize, \
     ConstRatePopulationSize
 from golem.core.optimisers.graph import OptGraph, OptNode
 from golem.core.optimisers.objective import Objective
 from golem.core.optimisers.opt_history_objects.individual import Individual
-from golem.core.utilities.sequence_iterator import SequenceIterator, fibonacci_sequence
+from golem.utilities.sequence_iterator import SequenceIterator
 
 
 def custom_objective():