Merge pull request #1515 from ales-erjavec/ward

[ENH] Enable Ward clustering in Hierarchical clustering widget

Author: tomazc

Orange/clustering/hierarchical.py

@@ -1,10 +1,14 @@
 from collections import namedtuple, deque
 from operator import attrgetter
 from itertools import chain, count
+from distutils.version import LooseVersion as _LooseVersion
+
 import heapq
 import numpy
 
 import scipy.cluster.hierarchy
+import scipy.spatial.distance
+
 from Orange.distance import Euclidean, PearsonR
 
 __all__ = ['HierarchicalClustering']
@@ -15,6 +19,17 @@
 WEIGHTED = "weighted"
 WARD = "ward"
 
+# Does scipy implement a O(n**2) NN chain algorithm?
+_HAS_NN_CHAIN = hasattr(scipy.cluster.hierarchy, "_hierarchy") and \
+                hasattr(scipy.cluster.hierarchy._hierarchy, "nn_chain")
+
+# Prior to 0.18 scipy.cluster.hierarchical's python interface disallowed
+# ward clustering from a precomputed distance matrix even though it's cython
+# implementation allowed it and was documented to support it (scipy issue 5220)
+_HAS_WARD_LINKAGE_FROM_DIST = \
+    _LooseVersion(scipy.__version__) >= _LooseVersion("0.18") and \
+    _HAS_NN_CHAIN
+
 
 def condensedform(X, mode="upper"):
     X = numpy.asarray(X)
@@ -86,7 +101,23 @@ def dist_matrix_linkage(matrix, linkage=AVERAGE):
     """
     # Extract compressed upper triangular distance matrix.
     distances = condensedform(matrix)
-    return scipy.cluster.hierarchy.linkage(distances, method=linkage)
+    if linkage == WARD and not _HAS_WARD_LINKAGE_FROM_DIST:
+        # Avoid `scipy.cluster.hierarchy.linkage` and dispatch to it's
+        # cython implementation directly.
+        # This the core of the scipy.cluster.hierarchy.linkage in
+        # scipy 0.16, 0.17. Assuming the branches are in bug fix mode
+        # only so this interface will not change.
+        y = numpy.asarray(distances, dtype=float)
+        scipy.spatial.distance.is_valid_y(y, throw=True)
+        N = scipy.spatial.distance.num_obs_y(y)
+        # allocate the output linkage matrix
+        Z = numpy.zeros((N - 1, 4))
+        # retrieve the correct method flag
+        method = scipy.cluster.hierarchy._cpy_euclid_methods["ward"]
+        scipy.cluster.hierarchy._hierarchy.linkage(y, Z, int(N), int(method))
+        return Z
+    else:
+        return scipy.cluster.hierarchy.linkage(distances, method=linkage)
 
 
 def dist_matrix_clustering(matrix, linkage=AVERAGE):
@@ -96,9 +127,7 @@ def dist_matrix_clustering(matrix, linkage=AVERAGE):
     :param Orange.misc.DistMatrix matrix:
     :param str linkage:
     """
-    # Extract compressed upper triangular distance matrix.
-    distances = condensedform(matrix)
-    Z = scipy.cluster.hierarchy.linkage(distances, method=linkage)
+    Z = dist_matrix_linkage(matrix, linkage=linkage)
     return tree_from_linkage(Z)
 
 
@@ -363,12 +392,11 @@ def item(node):
     heap = [item(tree)]
 
     while len(heap) < k:
-        key, cl = heapq.heappop(heap)
+        _, cl = heap[0]  # peek
         if cl.is_leaf:
             assert all(n.is_leaf for _, n in heap)
-            heapq.heappush(heap, (key, cl))
             break
-
+        key, cl = heapq.heappop(heap)
         left, right = cl.left, cl.right
         heapq.heappush(heap, item(left))
         heapq.heappush(heap, item(right))

Orange/tests/test_clustering_hierarchical.py

@@ -70,6 +70,16 @@ def test_prunning(self):
         pruned = hierarchical.prune(self.cluster, height=10)
         self.assertTrue(c.height >= 10 for c in hierarchical.preorder(pruned))
 
+        top = hierarchical.top_clusters(self.cluster, 3)
+        self.assertEqual(len(top), 3)
+
+        top = hierarchical.top_clusters(self.cluster, len(self.matrix))
+        self.assertEqual(len(top), len(self.matrix))
+        self.assertTrue(all(n.is_leaf for n in top))
+
+        top1 = hierarchical.top_clusters(self.cluster, len(self.matrix) + 1)
+        self.assertEqual(top1, top)
+
     def test_form(self):
         m = [[0, 2, 3, 4],
              [2, 0, 6, 7],
@@ -117,6 +127,14 @@ def score(root):
         self.assertGreater(score_unordered, score_ordered)
         self.assertEqual(score_ordered, 21.0)
 
+    def test_table_clustering(self):
+        table = Orange.data.Table(numpy.eye(3))
+        tree = hierarchical.data_clustering(table, linkage="single")
+        numpy.testing.assert_almost_equal(tree.value.height, numpy.sqrt(2))
+
+        tree = hierarchical.feature_clustering(table)
+        numpy.testing.assert_almost_equal(tree.value.height, 0.75)
+
 
 class TestTree(unittest.TestCase):
     def test_tree(self):

Orange/widgets/unsupervised/owdistancemap.py

@@ -263,7 +263,10 @@ class OWDistanceMap(widget.OWWidget):
     graph_name = "grid_widget"
 
     # Disable clustering for inputs bigger than this
-    _MaxClustering = 3000
+    if hierarchical._HAS_NN_CHAIN:
+        _MaxClustering = 25000
+    else:
+        _MaxClustering = 3000
 
     # Disable cluster leaf ordering for inputs bigger than this
     _MaxOrderedClustering = 1000

Orange/widgets/unsupervised/owhierarchicalclustering.py

@@ -3,11 +3,9 @@
 
 from collections import namedtuple, OrderedDict
 from itertools import chain
-from functools import reduce
 from contextlib import contextmanager
 
 import numpy
-import scipy.cluster.hierarchy
 
 from PyQt4.QtGui import (
     QGraphicsWidget, QGraphicsObject, QGraphicsLinearLayout, QGraphicsPathItem,
@@ -26,19 +24,17 @@
 from Orange.data.domain import filter_visible
 import Orange.misc
 from Orange.clustering.hierarchical import \
-    postorder, preorder, Tree, tree_from_linkage, leaves, prune, top_clusters
+    postorder, preorder, Tree, tree_from_linkage, dist_matrix_linkage, \
+    leaves, prune, top_clusters
 
 from Orange.widgets import widget, gui, settings
 from Orange.widgets.utils import colorpalette, itemmodels
 from Orange.widgets.io import FileFormat
 
 __all__ = ["OWHierarchicalClustering"]
 
-# In scipy 0.14 ward linkage cannot be computed from the distance
-# matrix alone, it requires the full data matrix (in 0.15 the whole
-# hierarchical clustering is/will be reimplemented and from the
-# looks of it will support ward from dist matrix).
-LINKAGE = ["Single", "Average", "Weighted", "Complete"]
+
+LINKAGE = ["Single", "Average", "Weighted", "Complete", "Ward"]
 
 
 def dendrogram_layout(tree, expand_leaves=False):
@@ -1003,14 +999,9 @@ def _update(self):
         distances = self.matrix
 
         if distances is not None:
-            # Convert to flat upper triangular distances
-            i, j = numpy.triu_indices(distances.shape[0], k=1)
-            distances = numpy.asarray(distances[i, j])
-
             method = LINKAGE[self.linkage].lower()
-            Z = scipy.cluster.hierarchy.linkage(
-                distances, method=method
-            )
+            Z = dist_matrix_linkage(distances, linkage=method)
+
             tree = tree_from_linkage(Z)
             self.linkmatrix = Z
             self.root = tree

Orange/widgets/visualize/owheatmap.py

@@ -27,6 +27,7 @@
 
 from Orange.widgets.unsupervised.owhierarchicalclustering import \
     DendrogramWidget
+
 from Orange.widgets.widget import Msg
 
 
@@ -398,7 +399,10 @@ class OWHeatMap(widget.OWWidget):
         (OrderedClustering, "Clustering with leaf ordering")
     ]
     # Disable clustering for inputs bigger than this
-    _MaxClustering = 3000
+    if hierarchical._HAS_NN_CHAIN:
+        _MaxClustering = 25000
+    else:
+        _MaxClustering = 3000
 
     # Disable cluster leaf ordering for inputs bigger than this
     _MaxOrderedClustering = 1000