Set random_seed for stohastic integration tests (#217)

* fix

* fix#2

* minor

* increase indulgence

* minor

* move set random seed

* minor

* add random seed for integration tests

* minor

* Try to log state hash after every action

* Set seed of urandom to all tests (by mocking) by adding global autouse fixture

* Revert "Set seed of urandom to all tests (by mocking) by adding global autouse fixture"

This reverts commit 03d2d1f.

* Revert "Try to log state hash after every action"

This reverts commit eeec7b4.

* Create conftest.py to fix seed and mock urandom before all tests

* Further resolve conflict of incorrect imports after moving `utilities` folder

* Use random.getrandbits for compatibility with python 3.8

* Add test that seed fully determines evolution process

* Remove check that evolution history differs for different seeds

* More robust history comparison

* Remove unnecessary utils.py file and references to `set_random_seed`

* Try debugging `test_hierarchy_pos` test internals

* Generate test picture

* Fix workflow to upload artifact

* Print networkx version in CI for debugging purposes

* Test requiring other version of networkX

* Revert "Print networkx version in CI for debugging purposes"

This reverts commit 94783b2.

* Revert "Fix workflow to upload artifact"

This reverts commit f3d490c.

* Revert "Generate test picture"

This reverts commit b723d98.

* Revert "Try debugging `test_hierarchy_pos` test internals"

This reverts commit 22fd62b.

* Fix redundant imports

---------

Co-authored-by: Vladimir Latypov <[email protected]>

golem/utilities/utilities.py

@@ -1,10 +1,9 @@
-import logging
 from typing import Optional
 
 import numpy as np
 from joblib import cpu_count
 
-from golem.utilities import random
+import random
 from golem.utilities.random import RandomStateHandler
 
 
@@ -22,6 +21,13 @@ def determine_n_jobs(n_jobs=-1, logger=None):
     return n_jobs
 
 
+def urandom_mock(n):
+    # os.random is the source of random used in the uuid library
+    # normally, it's „true“ random, but to stabilize tests,
+    # seeded `random` library is used instead.
+    return bytes(random.getrandbits(8) for _ in range(n))
+
+
 def set_random_seed(seed: Optional[int]):
     """ Sets random seed for evaluation of models. """
     if seed is not None:

requirements.txt

@@ -3,7 +3,7 @@ numpy>=1.16.0, !=1.24.0
 pandas>=1.3.0; python_version >='3.8'
 
 # Models and frameworks
-networkx>=2.4, !=2.7.*, !=2.8.1, !=2.8.2, !=2.8.3
+networkx>=2.4, !=2.7.*, !=2.8.1, !=2.8.2, !=2.8.3, != 3.3
 scipy>=1.7.3
 zss>=1.2.0
 

test/conftest.py

@@ -0,0 +1,11 @@
+from unittest.mock import patch
+import pytest
+from golem.utilities.utilities import urandom_mock, set_random_seed
+
+
+@pytest.fixture(autouse=True)
+def stabilize_random():
+    set_random_seed(42)
+
+    with patch('os.urandom', urandom_mock):
+        yield

test/integration/test_structural_analysis.py

@@ -63,7 +63,6 @@ def test_structural_analysis():
                                  requirements=requirements,
                                  path_to_save=path_to_save,
                                  is_visualize_per_iteration=False)
-
     optimized_graph, _ = sa.optimize(graph=opt_graph, n_jobs=1,
                                      max_iter=2)
 

test/integration/test_structure_search.py

@@ -35,14 +35,15 @@ def run_search(size: int, distance_function: Callable, timeout_min: int = 1) ->
 
 
 @pytest.mark.parametrize('target_sizes, distance_function, indulgence',
-                         [([10, 24], tree_edit_dist, 0.6),
-                          ([30], graph_size, 0.1)])
+                         [([10, 24], tree_edit_dist, 0.5),
+                          ([30], graph_size, 0.3)])
 def test_simple_targets_are_found(target_sizes, distance_function, indulgence):
     """ Checks if simple targets can be found within specified time. """
     for target_size in target_sizes:
         num_trials = 3
         distances = []
         for i in range(num_trials):
+            # to test num_trials different options
             distance, target_graph = run_search(target_size, distance_function=distance_function, timeout_min=1)
             distances.append(distance)
 

test/unit/utilities/random_seeds.py

@@ -0,0 +1,31 @@
+from examples.synthetic_graph_evolution.generators import generate_labeled_graph
+from examples.synthetic_graph_evolution.graph_search import graph_search_setup
+from golem.utilities.utilities import set_random_seed
+
+
+def test_random_seed_fully_determines_evolution_process():
+    """ Tests that random seed fully determines evolution process. """
+    # Setup graph search
+    target_graph = generate_labeled_graph('tree', 4, node_labels=['X', 'Y'])
+
+    def launch_with_seed(seed):
+        set_random_seed(seed)
+        optimizer, objective = graph_search_setup(
+            target_graph=target_graph,
+            num_iterations=3,
+            node_types=['X', 'Y'],
+            pop_size=3
+        )
+        optimizer.optimise(objective)
+        return optimizer.history
+
+    def history_equals(history1, history2):
+        def get_generation_ids(history):
+            return [[ind.graph.descriptive_id for ind in generation] for generation in history.generations]
+        return get_generation_ids(history1) == get_generation_ids(history2)
+
+    seed = 42
+    first_run = launch_with_seed(seed)
+    second_run = launch_with_seed(seed)
+
+    assert history_equals(first_run, second_run)