Merge pull request #143 from pavlin-policar/py39

Add Python 3.9 support

Author: pavlin-policar

azure-pipelines-release.yml

@@ -28,6 +28,9 @@ jobs:
         osx - python38:
           image.name: 'macos-10.14'
           python.version: '3.8'
+        osx - python39:
+          image.name: 'macos-10.14'
+          python.version: '3.9'
 
         windows - python36:
           image.name: 'vs2017-win2016'
@@ -38,6 +41,9 @@ jobs:
         windows - python38:
           image.name: 'vs2017-win2016'
           python.version: '3.8'
+        windows - python39:
+          image.name: 'vs2017-win2016'
+          python.version: '3.9'
 
     steps:
       - task: UsePythonVersion@0
@@ -71,10 +77,12 @@ jobs:
       - bash: python -m pip install -vv --force-reinstall --find-links dist openTSNE
         displayName: 'Install wheel'
 
-      - script: |
-          python -m pip install pynndescent
-          python -m pip install hnswlib
-        displayName: 'Install optional dependencies'
+      - script: pip install pynndescent
+        displayName: 'Install optional dependencies - pynndescent'
+        condition: ne(variables['python.version'], '3.9')
+
+      - script: pip install hnswlib
+        displayName: 'Install optional dependencies - hnswlib'
 
       - script: pytest -v
         timeoutInMinutes: 15
@@ -104,10 +112,16 @@ jobs:
       matrix:
         python36:
           python: '/opt/python/cp36-cp36m/bin'
+          python.version: '3.6'
         python37:
           python: '/opt/python/cp37-cp37m/bin'
+          python.version: '3.7'
         python38:
           python: '/opt/python/cp38-cp38/bin'
+          python.version: '3.8'
+        python39:
+          python: '/opt/python/cp39-cp39/bin'
+          python.version: '3.9'
 
     container:
       image: quay.io/pypa/manylinux1_x86_64:latest
@@ -132,10 +146,12 @@ jobs:
       - bash: mv openTSNE src
         displayName: 'Remove source files from path'
 
-      - script: |
-          $(python)/pip install pynndescent
-          $(python)/pip install hnswlib
-        displayName: 'Install optional dependencies'
+      - script: $(python)/pip install pynndescent
+        displayName: 'Install optional dependencies - pynndescent'
+        condition: ne(variables['python.version'], '3.9')
+
+      - script: $(python)/pip install hnswlib
+        displayName: 'Install optional dependencies - hnswlib'
 
       - script: $(python)/python -m pytest -v
         timeoutInMinutes: 15

azure-pipelines.yml

@@ -22,6 +22,9 @@ jobs:
       linux-python38:
         image.name: 'ubuntu-16.04'
         python.version: '3.8'
+      linux-python39:
+        image.name: 'ubuntu-16.04'
+        python.version: '3.9'
       osx-python36:
         image.name: 'macos-10.14'
         python.version: '3.6'
@@ -31,6 +34,9 @@ jobs:
       osx-python38:
         image.name: 'macos-10.14'
         python.version: '3.8'
+      osx-python39:
+        image.name: 'macos-10.14'
+        python.version: '3.9'
       windows-python36:
         image.name: 'vs2017-win2016'
         python.version: '3.6'
@@ -40,6 +46,9 @@ jobs:
       windows-python38:
         image.name: 'vs2017-win2016'
         python.version: '3.8'
+      windows-python39:
+        image.name: 'vs2017-win2016'
+        python.version: '3.9'
 
   steps:
   - task: UsePythonVersion@0
@@ -61,10 +70,12 @@ jobs:
   - script: pip install -vv .
     displayName: 'Install package'
 
-  - script: |
-      pip install pynndescent
-      pip install hnswlib
-    displayName: 'Install optional dependencies'
+  - script: pip install pynndescent
+    displayName: 'Install optional dependencies - pynndescent'
+    condition: ne(variables['python.version'], '3.9')
+
+  - script: pip install hnswlib
+    displayName: 'Install optional dependencies - hnswlib'
 
   # Since Python automatically adds `cwd` to `sys.path`, it's important we remove the local folder
   # containing our code from the working directory. Otherwise, the tests will use the local copy

openTSNE/affinity.py

@@ -9,6 +9,7 @@
 from openTSNE import _tsne
 from openTSNE import nearest_neighbors
 from openTSNE import utils
+from openTSNE.utils import is_package_installed
 
 log = logging.getLogger(__name__)
 
@@ -273,7 +274,8 @@ def check_perplexity(self, perplexity):
 def build_knn_index(
     data, method, k, metric, metric_params=None, n_jobs=1, random_state=None, verbose=False
 ):
-    if not sp.issparse(data) and metric in [
+    preferred_approx_method = nearest_neighbors.Annoy
+    if is_package_installed("pynndescent") and sp.issparse(data) and metric not in [
         "cosine",
         "euclidean",
         "manhattan",
@@ -283,8 +285,6 @@ def build_knn_index(
         "l2",
         "taxicab",
     ]:
-        preferred_approx_method = nearest_neighbors.Annoy
-    else:
         preferred_approx_method = nearest_neighbors.NNDescent
 
     if data.shape[0] < 1000:

openTSNE/utils.py

@@ -32,3 +32,14 @@ def func(*args, **kwargs):
             return f(*args, **kwargs)
         return func
     return wrapper
+
+
+def is_package_installed(libname):
+    """Check whether a python package is installed."""
+    import importlib
+
+    try:
+        importlib.import_module(libname)
+        return True
+    except ImportError:
+        return False

tests/test_nearest_neighbors.py

@@ -4,15 +4,12 @@
 import numpy as np
 import scipy.sparse as sp
 from scipy.spatial.distance import pdist, cdist, squareform
-import pynndescent
-import hnswlib
 from sklearn import datasets
 
-from numba import njit
-from numba.core.registry import CPUDispatcher
 from sklearn.utils import check_random_state
 
 from openTSNE import nearest_neighbors
+from openTSNE.utils import is_package_installed
 from .test_tsne import check_mock_called_with_kwargs
 
 
@@ -76,10 +73,6 @@ class TestAnnoy(KNNIndexTestMixin, unittest.TestCase):
     knn_index = nearest_neighbors.Annoy
 
 
-class TestHNSW(KNNIndexTestMixin, unittest.TestCase):
-    knn_index = nearest_neighbors.HNSW
-
-
 class TestBallTree(KNNIndexTestMixin, unittest.TestCase):
     knn_index = nearest_neighbors.BallTree
 
@@ -145,43 +138,37 @@ def manhattan(x, y):
             distances, true_distances_, err_msg="Distances do not match"
         )
 
-    def test_numba_compiled_callable_metric_same_result(self):
-        k = 15
 
-        knn_index = self.knn_index("manhattan", random_state=1)
-        knn_index.build(self.x1, k=k)
-        true_indices_, true_distances_ = knn_index.query(self.x2, k=k)
-
-        @njit(fastmath=True)
-        def manhattan(x, y):
-            result = 0.0
-            for i in range(x.shape[0]):
-                result += np.abs(x[i] - y[i])
-
-            return result
+@unittest.skipIf(not is_package_installed("hnswlib"), "`hnswlib`is not installed")
+class TestHNSW(KNNIndexTestMixin, unittest.TestCase):
+    knn_index = nearest_neighbors.HNSW
 
-        knn_index = self.knn_index(manhattan, random_state=1)
-        knn_index.build(self.x1, k=k)
-        indices, distances = knn_index.query(self.x2, k=k)
-        np.testing.assert_array_equal(
-            indices, true_indices_, err_msg="Nearest neighbors do not match"
-        )
-        np.testing.assert_allclose(
-            distances, true_distances_, err_msg="Distances do not match"
-        )
+    @classmethod
+    def setUpClass(cls):
+        global hnswlib
+        import hnswlib
 
 
+@unittest.skipIf(not is_package_installed("pynndescent"), "`pynndescent`is not installed")
 class TestNNDescent(KNNIndexTestMixin, unittest.TestCase):
     knn_index = nearest_neighbors.NNDescent
 
-    @patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent)
-    def test_random_state_being_passed_through(self, nndescent):
+    @classmethod
+    def setUpClass(cls):
+        global pynndescent, njit, CPUDispatcher
+
+        import pynndescent
+        from numba import njit
+        from numba.core.registry import CPUDispatcher
+
+    def test_random_state_being_passed_through(self):
         random_state = 1
-        knn_index = nearest_neighbors.NNDescent("euclidean", random_state=random_state)
-        knn_index.build(self.x1, k=30)
+        with patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent) as nndescent:
+            knn_index = nearest_neighbors.NNDescent("euclidean", random_state=random_state)
+            knn_index.build(self.x1, k=30)
 
-        nndescent.assert_called_once()
-        check_mock_called_with_kwargs(nndescent, {"random_state": random_state})
+            nndescent.assert_called_once()
+            check_mock_called_with_kwargs(nndescent, {"random_state": random_state})
 
     def test_uncompiled_callable_is_compiled(self):
         knn_index = nearest_neighbors.NNDescent("manhattan")
@@ -245,47 +232,47 @@ def manhattan(x, y):
             distances, true_distances_, err_msg="Distances do not match"
         )
 
-    @patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent)
-    def test_building_with_lt15_builds_proper_graph(self, nndescent):
-        knn_index = nearest_neighbors.NNDescent("euclidean")
-        indices, distances = knn_index.build(self.x1, k=10)
+    def test_building_with_lt15_builds_proper_graph(self):
+        with patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent) as nndescent:
+            knn_index = nearest_neighbors.NNDescent("euclidean")
+            indices, distances = knn_index.build(self.x1, k=10)
 
-        self.assertEqual(indices.shape, (self.x1.shape[0], 10))
-        self.assertEqual(distances.shape, (self.x1.shape[0], 10))
-        self.assertFalse(np.all(indices[:, 0] == np.arange(self.x1.shape[0])))
+            self.assertEqual(indices.shape, (self.x1.shape[0], 10))
+            self.assertEqual(distances.shape, (self.x1.shape[0], 10))
+            self.assertFalse(np.all(indices[:, 0] == np.arange(self.x1.shape[0])))
 
         # Should be called with 11 because nearest neighbor in pynndescent is itself
         check_mock_called_with_kwargs(nndescent, dict(n_neighbors=11))
 
-    @patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent)
-    def test_building_with_gt15_calls_query(self, nndescent):
-        nndescent.query = MagicMock(wraps=nndescent.query)
-        knn_index = nearest_neighbors.NNDescent("euclidean")
-        indices, distances = knn_index.build(self.x1, k=30)
-
-        self.assertEqual(indices.shape, (self.x1.shape[0], 30))
-        self.assertEqual(distances.shape, (self.x1.shape[0], 30))
-        self.assertFalse(np.all(indices[:, 0] == np.arange(self.x1.shape[0])))
-
-        # The index should be built with 15 neighbors
-        check_mock_called_with_kwargs(nndescent, dict(n_neighbors=15))
-        # And subsequently queried with the correct number of neighbors. Check
-        # for 31 neighbors because query will return the original point as well,
-        # which we don't consider.
-        check_mock_called_with_kwargs(nndescent.query, dict(k=31))
-
-    @patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent)
-    def test_runs_with_correct_njobs_if_dense_input(self, nndescent):
-        knn_index = nearest_neighbors.NNDescent("euclidean", n_jobs=2)
-        knn_index.build(self.x1, k=5)
-        check_mock_called_with_kwargs(nndescent, dict(n_jobs=2))
-
-    @patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent)
-    def test_runs_with_correct_njobs_if_sparse_input(self, nndescent):
-        x_sparse = sp.csr_matrix(self.x1)
-        knn_index = nearest_neighbors.NNDescent("euclidean", n_jobs=2)
-        knn_index.build(x_sparse, k=5)
-        check_mock_called_with_kwargs(nndescent, dict(n_jobs=2))
+    def test_building_with_gt15_calls_query(self):
+        with patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent) as nndescent:
+            nndescent.query = MagicMock(wraps=nndescent.query)
+            knn_index = nearest_neighbors.NNDescent("euclidean")
+            indices, distances = knn_index.build(self.x1, k=30)
+
+            self.assertEqual(indices.shape, (self.x1.shape[0], 30))
+            self.assertEqual(distances.shape, (self.x1.shape[0], 30))
+            self.assertFalse(np.all(indices[:, 0] == np.arange(self.x1.shape[0])))
+
+            # The index should be built with 15 neighbors
+            check_mock_called_with_kwargs(nndescent, dict(n_neighbors=15))
+            # And subsequently queried with the correct number of neighbors. Check
+            # for 31 neighbors because query will return the original point as well,
+            # which we don't consider.
+            check_mock_called_with_kwargs(nndescent.query, dict(k=31))
+
+    def test_runs_with_correct_njobs_if_dense_input(self):
+        with patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent) as nndescent:
+            knn_index = nearest_neighbors.NNDescent("euclidean", n_jobs=2)
+            knn_index.build(self.x1, k=5)
+            check_mock_called_with_kwargs(nndescent, dict(n_jobs=2))
+
+    def test_runs_with_correct_njobs_if_sparse_input(self):
+        with patch("pynndescent.NNDescent", wraps=pynndescent.NNDescent) as nndescent:
+            x_sparse = sp.csr_matrix(self.x1)
+            knn_index = nearest_neighbors.NNDescent("euclidean", n_jobs=2)
+            knn_index.build(x_sparse, k=5)
+            check_mock_called_with_kwargs(nndescent, dict(n_jobs=2))
 
     def test_random_cluster_when_invalid_indices(self):
         class MockIndex:

tests/test_tsne.py

@@ -15,6 +15,7 @@
 from openTSNE.affinity import PerplexityBasedNN
 from openTSNE.nearest_neighbors import NNDescent
 from openTSNE.tsne import kl_divergence_bh, kl_divergence_fft
+from openTSNE.utils import is_package_installed
 
 np.random.seed(42)
 affinity.log.setLevel(logging.ERROR)
@@ -216,6 +217,7 @@ def test_partial_embedding_optimize(self, param_name, param_value, gradient_desc
         check_call_contains_kwargs(gradient_descent.mock_calls[0], params)
 
     @check_params({"metric": set(NNDescent.VALID_METRICS) - {"mahalanobis"}})
+    @unittest.skipIf(not is_package_installed("pynndescent"), "`pynndescent`is not installed")
     @patch("pynndescent.NNDescent")
     def test_nndescent_distances(self, param_name, metric, nndescent: MagicMock):
         """Distance metrics should be properly passed down to NN descent"""
@@ -234,6 +236,7 @@ def test_nndescent_distances(self, param_name, metric, nndescent: MagicMock):
         self.assertEqual(nndescent.call_count, 1)
         check_call_contains_kwargs(nndescent.mock_calls[0], {"metric": metric})
 
+    @unittest.skipIf(not is_package_installed("pynndescent"), "`pynndescent`is not installed")
     @patch("pynndescent.NNDescent")
     def test_nndescent_mahalanobis_distance(self, nndescent: MagicMock):
         """Distance metrics and additional params should be correctly passed down to NN descent"""
@@ -576,12 +579,12 @@ def test_random_state_parameter_is_propagated_random_init_exact(self, init, neig
         check_mock_called_with_kwargs(neighbors, {"random_state": random_state})
 
     @patch("openTSNE.initialization.pca", wraps=openTSNE.initialization.pca)
-    @patch("openTSNE.nearest_neighbors.NNDescent", wraps=openTSNE.nearest_neighbors.NNDescent)
+    @patch("openTSNE.nearest_neighbors.Annoy", wraps=openTSNE.nearest_neighbors.Annoy)
     def test_random_state_parameter_is_propagated_pca_init_approx(self, init, neighbors):
         random_state = 1
 
         tsne = openTSNE.TSNE(
-            neighbors="pynndescent",
+            neighbors="approx",
             initialization="pca",
             random_state=random_state,
         )