Make singa-cpu runnable on Ubuntu 24.04

.gitignore

@@ -20,6 +20,10 @@ __pycache__
 *.cxx
 
 build/
+src/api/config.i
+src/api/singa_wrap.cpp
+dist/
+include/singa/singa_config.h
 thirdparty/*
 !thirdparty/install.sh
 test/samples/
@@ -30,3 +34,6 @@ test/samples/
 doc/_build/*
 doc/en/docs/model_zoo/
 cmake-build-debug/*
+
+# Python package
+python/singa.egg-info/

examples/cnn_ms/pkg_model_code/model.py

@@ -22,12 +22,17 @@
 
 import os
 import gc
+import sys
 import time
 import json
 import zipfile
 import numpy as np
 from functools import wraps
-from collections import Iterable
+
+if sys.version_info < (3, 10):
+    from collections import Iterable
+else:
+    from collections.abc import Iterable
 
 from singa import tensor
 from singa import autograd

examples/demos/Classification/BloodMnist/transforms.py

@@ -95,9 +95,9 @@ def forward(self, pic):
         img = np.transpose(img, (2, 0, 1))
 
         if img.dtype == np.uint8:
-            return np.array(np.float32(img)/255.0, dtype=np.float)
+            return np.array(np.float32(img)/255.0, dtype=np.float64)
         else:
-            return np.float(img)
+            return np.float64(img)
 
     def __repr__(self):
         return self.__class__.__name__ + '()'
@@ -139,7 +139,7 @@ def forward(self, img: np.ndarray):
         if not isinstance(img, np.ndarray):
             raise TypeError('Input img should be a numpy array. Got {}.'.format(type(img)))
 
-        if not img.dtype == np.float:
+        if not img.dtype == float:
             raise TypeError('Input array should be a float array. Got {}.'.format(img.dtype))
 
         if img.ndim < 3:

examples/healthcare/Hematologic_Disease/transforms.py

@@ -95,9 +95,9 @@ def forward(self, pic):
         img = np.transpose(img, (2, 0, 1))
 
         if img.dtype == np.uint8:
-            return np.array(np.float32(img)/255.0, dtype=np.float)
+            return np.array(np.float32(img)/255.0, dtype=np.float64)
         else:
-            return np.float(img)
+            return np.float64(img)
 
     def __repr__(self):
         return self.__class__.__name__ + '()'
@@ -139,7 +139,7 @@ def forward(self, img: np.ndarray):
         if not isinstance(img, np.ndarray):
             raise TypeError('Input img should be a numpy array. Got {}.'.format(type(img)))
 
-        if not img.dtype == np.float:
+        if not img.dtype == float:
             raise TypeError('Input array should be a float array. Got {}.'.format(img.dtype))
 
         if img.ndim < 3:

examples/healthcare/data/bloodmnist.py

@@ -98,9 +98,9 @@ def forward(self, pic):
         img = np.transpose(img, (2, 0, 1))
 
         if img.dtype == np.uint8:
-            return np.array(np.float32(img) / 255.0, dtype=np.float)
+            return np.array(np.float32(img) / 255.0, dtype=np.float64)
         else:
-            return np.float(img)
+            return np.float64(img)
 
     def __repr__(self):
         return self.__class__.__name__ + '()'
@@ -142,7 +142,7 @@ def forward(self, img: np.ndarray):
         if not isinstance(img, np.ndarray):
             raise TypeError('Input img should be a numpy array. Got {}.'.format(type(img)))
 
-        if not img.dtype == np.float:
+        if not img.dtype == float:
             raise TypeError('Input array should be a float array. Got {}.'.format(img.dtype))
 
         if img.ndim < 3:

examples/model_selection/Trails/internal/ml/model_selection/src/eva_engine/phase1/algo/singa_ms/cnn_ms/pkg_model_code/model.py

@@ -22,12 +22,18 @@
 
 import os
 import gc
+import sys
 import time
 import json
 import zipfile
 import numpy as np
 from functools import wraps
-from collections import Iterable
+
+if sys.version_info < (3, 10):
+    from collections import Iterable
+else:
+    from collections.abc import Iterable
+
 
 from singa import tensor
 from singa import autograd

examples/model_selection/Trails/singa_pkg_code/model.py

@@ -22,17 +22,17 @@
 
 import os
 import gc
+import sys
 import time
 import json
 import zipfile
 import numpy as np
 from functools import wraps
-# from collections import Iterable
 
-try:
-    from collections.abc import Iterable
-except ImportError:
+if sys.version_info < (3, 10):
     from collections import Iterable
+else:
+    from collections.abc import Iterable
 
 from singa import tensor
 from singa import autograd

examples/model_selection/Trails/singa_pkg_code/tensor.py

@@ -855,7 +855,7 @@ def from_numpy(np_array, dev=None):
     '''
     assert type(np_array) is np.ndarray, 'Must input numpy array'
     # convert to float32 array
-    if np_array.dtype == np.float64 or np_array.dtype == np.float:
+    if np_array.dtype == np.float64:
         np_array = np_array.astype(np.float32)
 
     if np_array.dtype == np.int64 or np_array.dtype == int:

python/CMakeLists.txt

@@ -94,14 +94,8 @@ file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/singa/proto)
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/rafiki)
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/api)
 
-IF(USE_PYTHON3)
-	SET(SWIG_PYTHON3 "-py3")
-ELSE()
-	SET(SWIG_PYTHON3 "")
-ENDIF()
-
 execute_process(
-    COMMAND swig -c++ -python ${SWIG_PYTHON3} -I${CMAKE_SOURCE_DIR}/include
+    COMMAND swig -c++ -python -I${CMAKE_SOURCE_DIR}/include
     -outdir ${CMAKE_BINARY_DIR}/python/singa
     -o ${CMAKE_BINARY_DIR}/src/api/singa_wrap.cxx
     ${CMAKE_SOURCE_DIR}/src/api/singa.i)

python/singa/model.py

@@ -21,13 +21,18 @@
 '''
 
 import os
+import sys
 import gc
 import time
 import json
 import zipfile
 import numpy as np
 from functools import wraps
-from collections import Iterable
+
+if sys.version_info < (3, 10):
+    from collections import Iterable
+else:
+    from collections.abc import Iterable
 
 from singa import tensor
 from singa import autograd

python/singa/sonnx.py

@@ -19,15 +19,48 @@
 
 from __future__ import division
 
-import numpy as np
+import collections
+import sys
+import warnings
+import pkg_resources
 
+if sys.version_info < (3, 10):
+    from collections import Iterable
+else:
+    from collections.abc import Iterable
+
+import numpy as np
 import onnx
-import onnx.utils
 from onnx.backend.base import Backend, BackendRep
+
 from onnx import (checker, helper, numpy_helper, GraphProto, NodeProto,
-                  TensorProto, OperatorSetIdProto, optimizer, mapping,
+                  TensorProto, OperatorSetIdProto, mapping,
                   shape_inference)
-import warnings
+
+# ref: https://github.com/onnx/onnx/issues/582
+# ref: https://github.com/onnx/optimizer/blob/master/onnxoptimizer/__init__.py
+ONNX_VERSION = pkg_resources.parse_version(onnx.__version__)
+if ONNX_VERSION < pkg_resources.parse_version('1.9.0'):
+    from onnx import optimizer
+    import onnx.utils
+
+    OPTIMIZE_MODEL_FUNC = onnx.utils.polish_model
+else:
+    import onnxoptimizer as optimizer
+
+    def polish_model(model):  # type: (ModelProto) -> ModelProto
+        '''
+            This function combines several useful utility functions together.
+        '''
+        onnx.checker.check_model(model)
+        onnx.helper.strip_doc_string(model)
+        model = onnx.shape_inference.infer_shapes(model)
+        model = optimizer.optimize(model)
+        onnx.checker.check_model(model)
+        return model
+
+    OPTIMIZE_MODEL_FUNC = polish_model
+
 
 from . import device
 from . import autograd
@@ -37,7 +70,6 @@
 from . import utils
 from . import singa_wrap as singa
 
-import collections
 OrderedDict = collections.OrderedDict
 namedtuple = collections.namedtuple
 
@@ -57,7 +89,7 @@
     np.dtype('complex128'): None,
     np.dtype('uint32'): None,
     np.dtype('uint64'): None,
-    np.dtype(np.object): None
+    np.dtype(object): None
 }
 
 
@@ -900,7 +932,7 @@ def singa_op_to_onnx_node(cls, op, op_t):
         else:
             translator = cls._common_singa_tensor_to_onnx_node
         nodes = translator(op, op_t)
-        if not isinstance(nodes, collections.Iterable):
+        if not isinstance(nodes, Iterable):
             nodes = [nodes]
         nodes = [node for node in nodes if node is not None]
         return nodes
@@ -1826,7 +1858,7 @@ def _run_node(cls, operator, inputs):
             list, the output
         """
         outputs = operator(*inputs)
-        if not isinstance(outputs, collections.Iterable):
+        if not isinstance(outputs, Iterable):
             outputs = [outputs]
         return outputs
 
@@ -1920,7 +1952,7 @@ def prepare(cls, model, device='CPU', **kwargs):
         super(SingaBackend, cls).prepare(model, device, **kwargs)
         # optimize and infer the shape of the model
         try:
-            model = onnx.utils.polish_model(model)
+            model = OPTIMIZE_MODEL_FUNC(model)
         except IndexError as err:
             model = shape_inference.infer_shapes(model)
 

python/singa/tensor.py

@@ -372,7 +372,7 @@ def copy_from_numpy(self, np_array, offset=0):
             self.data.CopyFloatDataFromHostPtr(np_array)
         elif dt == np.float16:
             self.data.CopyHalfFloatDataFromHostPtr(np_array)
-        elif dt == np.int or dt == np.int32:
+        elif dt == np.int64 or dt == np.int32:
             self.data.CopyIntDataFromHostPtr(np_array)
         else:
             raise NotImplementedError('Not implemented yet for ', dt)
@@ -886,10 +886,10 @@ def from_numpy(np_array, dev=None):
     '''
     assert type(np_array) is np.ndarray, 'Must input numpy array'
     # convert to float32 array
-    if np_array.dtype == np.float64 or np_array.dtype == np.float:
+    if np_array.dtype == np.float64 or np_array.dtype == float:
         np_array = np_array.astype(np.float32)
 
-    if np_array.dtype == np.int64 or np_array.dtype == np.int:
+    if np_array.dtype == np.int64 or np_array.dtype == int:
         np_array = np_array.astype(np.int32)
 
     if np_array.dtype == np.float32:
@@ -1791,7 +1791,7 @@ def copy_from_numpy(data, np_array):
         data.CopyFloatDataFromHostPtr(np_array)
     elif dt == np.float16:
         data.CopyHalfFloatDataFromHostPtr(np_array)
-    elif dt == np.int or dt == np.int32:
+    elif dt == int or dt == np.int32:
         data.CopyIntDataFromHostPtr(np_array)
     else:
         raise NotImplementedError('Not implemented yet for ', dt)

setup.py

@@ -353,7 +353,7 @@ def finalize_options(self):
         print('build temp', self.build_temp)
         print('build lib', self.build_lib)
         super(custom_build_ext, self).finalize_options()
-        self.swig_opts = '-py3 -outdir {}/singa/'.format(self.build_lib).split()
+        self.swig_opts = '-outdir {}/singa/'.format(self.build_lib).split()
         print('build temp', self.build_temp)
         print('build lib', self.build_lib)
 

test/python/test_tensor.py

@@ -208,7 +208,7 @@ def test_tensor_inplace_api(self):
         self.assertTrue(y is x)
 
     def test_numpy_convert(self):
-        a = np.asarray([[1, 0, 0], [0, 1, 0]], dtype=np.int)
+        a = np.asarray([[1, 0, 0], [0, 1, 0]], dtype=np.int64)
         t = tensor.from_numpy(a)
         b = tensor.to_numpy(t)
         self.assertEqual(np.sum(a - b), 0)