Resolves #81 (#84)

CinquilCinquil · web-flow · commit 9d80743616fb · 2024-07-05T10:50:26.000-03:00
* chore(issue-81): Create a base class for saving and loading #81 * refactor: Adjusted to reviewer's suggestions . renamed black_list to block_list . renamed pickle_block_list to attr_block_list . removed SavableAttr class . left only get_full_persistence_path of the get_full_x_persistence_path methods * refactor: Adjusted to some of reviewer's suggestions #84 * refactor: More alterations #84 * fix: Replaced attr with key * refactor: Adjusted to some of reviewer's suggestions * refactor: Changed class and file name to Persistence #84 * feat: Create class PersistencePickle .Persistence class is now abstract .Furthermore, pickle related behaviour was moved to PersistencePickle * feat: Generalize some of Persistence's methods #84 * refactor: Remove unused import #84 * refactor: Rename _save to _simple_save #84 * test(issue-81): Add unit tests for Persistence #84 * test(issue-81): Add unit tests for PersistencePickle #84 * test(issue-81): Mock simple_save and load #84 * refactor(issue-81): Adjusted to reviewer's suggestions #84 * refactor(issue-81): Adjusted to lint's suggestions #84 * refactor(issue-81): Tests now in "given when then" format #84
diff --git a/tests/units/base/__init__.py b/tests/units/base/__init__.py
diff --git a/tests/units/base/test_persistence.py b/tests/units/base/test_persistence.py
@@ -0,0 +1,103 @@
+import os
+import unittest
+from abc import ABCMeta
+
+from urnai.base.persistence import Persistence
+
+
+class FakePersistence(Persistence):
+    Persistence.__abstractmethods__ = set()
+
+    def __init__(self, threaded_saving=False):
+        super().__init__(threaded_saving)
+
+class TestPersistence(unittest.TestCase):
+
+    def test_abstract_methods(self):
+
+        # GIVEN
+        fake_persistence = FakePersistence()
+
+        # WHEN
+        _simple_save_return = fake_persistence._simple_save(".")
+        load_return = fake_persistence.load(".")
+        _get_dict_return = fake_persistence._get_dict()
+        _get_attributes_return = fake_persistence._get_attributes()
+
+        # THEN
+        assert isinstance(Persistence, ABCMeta)
+        assert _simple_save_return is None
+        assert load_return is None
+        assert _get_dict_return is None
+        assert _get_attributes_return is None
+
+    def test_get_default_save_stamp(self):
+
+        # GIVEN
+        fake_persistence = FakePersistence()
+
+        # WHEN
+        _get_def_save_stamp_return = fake_persistence._get_default_save_stamp()
+
+        # THEN
+        self.assertEqual(_get_def_save_stamp_return,
+                          fake_persistence.__class__.__name__ + '_')
+
+    def test_get_full_persistance_path(self):
+
+        # GIVEN
+        fake_persistence = FakePersistence()
+        persist_path = "test"
+        
+        # WHEN
+        get_full_pers_path_return = fake_persistence.get_full_persistance_path(
+            persist_path)
+
+        # THEN
+        self.assertEqual(get_full_pers_path_return,
+            persist_path + os.path.sep + 
+            fake_persistence._get_default_save_stamp())
+        
+    def test_save(self):
+
+        # GIVEN
+        fake_persistence = FakePersistence()
+        persist_path = "test"
+
+        # WHEN
+        save_return = fake_persistence.save(persist_path)
+        fake_persistence_threaded = FakePersistence(threaded_saving=True)
+        fake_persistence_threaded.save(persist_path)
+
+        # THEN
+        assert save_return is None
+        for process in fake_persistence_threaded.processes:
+            assert process.is_alive() is True
+
+    def test_threaded_save(self):
+        
+        # GIVEN
+        fake_persistence = FakePersistence(threaded_saving=True)
+        persist_path = "test"
+
+        # WHEN
+        fake_persistence.save(persist_path)
+
+        # THEN
+        for process in fake_persistence.processes:
+            assert process.is_alive() is True
+
+    def test_restore_attributes(self):
+
+        # GIVEN
+        fake_persistence = FakePersistence()
+        dict_to_restore = {"TestAttribute1": 314, "TestAttribute2": "string"}
+
+        # WHEN
+        fake_persistence.attr_block_list = ["TestAttribute2"]
+        fake_persistence._restore_attributes(dict_to_restore)
+
+        # THEN
+        assert hasattr(fake_persistence, "TestAttribute1") is True
+        assert hasattr(fake_persistence, "TestAttribute2") is False
+        self.assertEqual(fake_persistence.TestAttribute1, 314)
diff --git a/tests/units/base/test_persistence_pickle.py b/tests/units/base/test_persistence_pickle.py
@@ -0,0 +1,66 @@
+import unittest
+from unittest.mock import patch
+
+from urnai.base.persistence_pickle import PersistencePickle
+
+
+class FakePersistencePickle(PersistencePickle):
+    def __init__(self, threaded_saving=False):
+        super().__init__(threaded_saving)
+
+class TestPersistence(unittest.TestCase):
+
+    @patch('urnai.base.persistence_pickle.PersistencePickle._simple_save')
+    def test_simple_save(self, mock_simple_save):
+
+        # GIVEN
+        fake_persistence_pickle = FakePersistencePickle()
+        persist_path = "test_simple_save"
+
+        # WHEN
+        mock_simple_save.return_value = "return_value"
+        simple_save_return = fake_persistence_pickle._simple_save(persist_path)
+
+        # THEN
+        self.assertEqual(simple_save_return, "return_value")
+
+    @patch('urnai.base.persistence_pickle.PersistencePickle.load')
+    def test_load(self, mock_load):
+        """
+        This method creates a FakePersistencePickle with certain values
+        and saves it (state1). After that, it changes the object's
+        attributes (state2) and loads it back to state1.
+        """
+
+        # GIVEN
+        fake_persistence_pickle = FakePersistencePickle()
+        persist_path = "test_load"
+        mock_load.return_value = "return_value"
+
+        # WHEN
+        load_return = fake_persistence_pickle.load(persist_path)
+
+        # THEN
+        self.assertEqual(load_return, "return_value")
+
+    def test_get_attributes(self):
+
+        # GIVEN
+        fake_persistence_pickle = FakePersistencePickle()
+
+        # WHEN
+        return_list = fake_persistence_pickle._get_attributes()
+
+        # THEN
+        self.assertEqual(return_list, ['threaded_saving'])
+
+    def test_get_dict(self):
+
+        # GIVEN
+        fake_persistence_pickle = FakePersistencePickle()
+
+        # WHEN
+        return_dict = fake_persistence_pickle._get_dict()
+
+        # THEN
+        self.assertEqual(return_dict, {"threaded_saving": False})
diff --git a/urnai/base/persistence.py b/urnai/base/persistence.py
@@ -0,0 +1,78 @@
+import os
+from abc import ABC, abstractmethod
+from multiprocessing import Process
+
+
+class Persistence(ABC):
+    """
+    This interface represents the concept of a class that can be saved to disk.
+    The heir class should define a constant or attribute as a default filename
+	to save on disk.
+    """
+
+    def __init__(self, threaded_saving=False):
+        self.threaded_saving = threaded_saving
+        self.attr_block_list = []
+        self.processes = []
+
+    def _get_default_save_stamp(self):
+        """
+        This method returns the default
+        file name that should be used while
+        persisting the object.
+        """
+        return self.__class__.__name__ + '_'
+
+    def get_full_persistance_path(self, persist_path):
+        """This method returns the default persistance path."""
+        return persist_path + os.path.sep + self._get_default_save_stamp()
+
+    def save(self, savepath):
+        if self.threaded_saving:
+            self._threaded_save(savepath)
+        else:
+            self._simple_save(savepath)
+
+    def _threaded_save(self, savepath):
+        """
+        This method saves pickleable
+        elements in a separate thread
+        """
+        new_process = Process(target=self._simple_save, args=(savepath,))
+        self.processes.append(new_process)
+        new_process.start()
+
+    @abstractmethod
+    def _simple_save(self, savepath):
+        """
+        This method handles logic related
+        to non-threaded saving in the child class
+        """
+        ...
+
+    @abstractmethod
+    def load(self, savepath):
+        ...
+
+    @abstractmethod
+    def _get_dict(self):
+        """
+        This method returns a dict with
+        all the attributes that will be
+        saved
+        """
+        ...
+
+    @abstractmethod
+    def _get_attributes(self):
+        """
+        This method returns all of the
+        attribute names that can be saved
+        except those in blocklist
+        """
+        ...
+
+    def _restore_attributes(self, dict_to_restore):
+        for key in dict_to_restore:
+            if key not in self.attr_block_list:   
+                setattr(self, key, dict_to_restore[key])
diff --git a/urnai/base/persistence_pickle.py b/urnai/base/persistence_pickle.py
@@ -0,0 +1,114 @@
+import os
+import pickle
+import tempfile
+
+from urnai.base.persistence import Persistence
+
+
+class PersistencePickle(Persistence):
+    """
+    This interface represents the concept of a class that can be saved to disk.
+    The heir class should define a constant or attribute as a default filename
+	to save on disk.
+    """
+
+    def __init__(self, threaded_saving=False):
+        super().__init__(threaded_saving)
+
+    def _simple_save(self, persist_path):
+        """
+        This method saves our instance
+        using pickle.
+
+        First it checks which attributes should be
+        saved using pickle, the ones which are not
+        are backuped.
+
+        Then all unpickleable attributes are set to None
+        and the object is pickled.
+
+        Finally the nulled attributes are
+        restored.
+        """
+        path = self.get_full_persistance_path(persist_path)
+		
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+		
+        with open(path, 'wb') as pickle_out:
+            pickle.dump(self._get_dict(), pickle_out)
+
+    def load(self, persist_path):
+        """
+        This method loads a list instance
+        saved by pickle.
+        """
+        pickle_path = self.get_full_persistance_path(persist_path)
+        exists_pickle = os.path.isfile(pickle_path)
+        
+        if exists_pickle and os.path.getsize(pickle_path) > 0:
+            with open(pickle_path, 'rb') as pickle_in:
+                pickle_dict = pickle.load(pickle_in)
+                self._restore_attributes(pickle_dict)
+
+    def _get_attributes(self):
+        """
+        This method returns a list of pickeable attributes.
+		If you wish to block one particular pickleable attribute, put it
+        in self.attr_block_list as a string.
+        """
+        if not hasattr(self, 'attr_block_list') or self.attr_block_list is None:
+            self.attr_block_list = []
+
+        attr_block_list = self.attr_block_list + ['attr_block_list', 'processes']
+
+        full_attr_list = [attr for attr in dir(self) if not attr.startswith('__')
+                          and not callable(getattr(self, attr))
+                          and attr not in attr_block_list
+                          and 'abc' not in attr]
+        
+        pickleable_list = []
+
+        for key in full_attr_list:
+            try:
+                with tempfile.NamedTemporaryFile() as tmp_file:
+                    pickle.dump(getattr(self, key), tmp_file)
+                    tmp_file.flush()
+                
+                pickleable_list.append(key)
+                
+            except pickle.PicklingError:
+                continue
+            
+            except TypeError as type_error:
+                if ("can't pickle" not in str(type_error) and
+                 'cannot pickle' not in str(type_error)):
+                    raise TypeError() from type_error
+                continue
+            
+            except NotImplementedError as notimpl_error:
+                if (str(notimpl_error) != 
+                'numpy() is only available when eager execution is enabled.'):
+                    raise NotImplementedError() from notimpl_error
+                continue
+            
+            except AttributeError as attr_error:
+                if ("Can't pickle" not in str(attr_error) and 
+                "object has no attribute '__getstate__'" not in
+                str(attr_error)):
+                    raise AttributeError() from attr_error
+                continue
+            
+            except ValueError as value_error:
+                if 'ctypes objects' not in str(value_error):
+                    raise ValueError() from value_error
+                continue
+
+        return pickleable_list
+
+    def _get_dict(self):
+        pickleable_attr_dict = {}
+
+        for attr in self._get_attributes():
+            pickleable_attr_dict[attr] = getattr(self, attr)
+
+        return pickleable_attr_dict
