Update fairlens dependencies

Update fairlens dependencies

Author: gsolard

setup.cfg

@@ -37,7 +37,6 @@ install_requires =
     matplotlib>=2.1.0
     seaborn>=0.11.1
     dcor>=0.5.3
-    pyemd==0.5.1
 
 [options.packages.find]
 where = src
@@ -53,6 +52,7 @@ dev =
 test =
     pytest>=6
     pytest-cov>=2
+    pyemd>=1.0.0
 
 doc =
     sphinx==3.5.4

src/fairlens/metrics/__init__.py

@@ -29,7 +29,7 @@
 )
 
 from .significance import (  # isort:skip
-    binom_test,
+    binomtest,
     binominal_proportion_p_value,
     binominal_proportion_interval,
     bootstrap_binned_statistic,
@@ -61,7 +61,7 @@
     "r2_mcfadden",
     "kruskal_wallis",
     "kruskal_wallis_boolean",
-    "binom_test",
+    "binomtest",
     "binominal_proportion_p_value",
     "binominal_proportion_interval",
     "bootstrap_binned_statistic",

src/fairlens/metrics/correlation.py

@@ -184,9 +184,11 @@ def distance_nn_correlation(sr_a: pd.Series, sr_b: pd.Series) -> float:
     warnings.filterwarnings(action="ignore", category=UserWarning)
 
     if sr_a.size < sr_b.size:
-        sr_a = sr_a.append(pd.Series(sr_a.mean()).repeat(sr_b.size - sr_a.size), ignore_index=True)
+        new_serie = pd.Series(sr_a.mean()).repeat(sr_b.size - sr_a.size)
+        sr_a = pd.concat([sr_a, new_serie], ignore_index=True)
     elif sr_a.size > sr_b.size:
-        sr_b = sr_b.append(pd.Series(sr_b.mean()).repeat(sr_a.size - sr_b.size), ignore_index=True)
+        new_serie = pd.Series(sr_b.mean()).repeat(sr_a.size - sr_b.size)
+        sr_b = pd.concat([sr_b, new_serie], ignore_index=True)
 
     return dcor.distance_correlation(sr_a, sr_b)
 
@@ -222,9 +224,11 @@ def distance_cn_correlation(sr_a: pd.Series, sr_b: pd.Series) -> float:
 
             # Handle groups with a different number of elements.
             if sr_i.size < sr_j.size:
-                sr_i = sr_i.append(sr_i.sample(sr_j.size - sr_i.size, replace=True), ignore_index=True)
+                new_serie = sr_i.sample(sr_j.size - sr_i.size, replace=True)
+                sr_i = pd.concat([sr_i, new_serie], ignore_index=True)
             elif sr_i.size > sr_j.size:
-                sr_j = sr_j.append(sr_j.sample(sr_i.size - sr_j.size, replace=True), ignore_index=True)
+                new_serie = sr_j.sample(sr_i.size - sr_j.size, replace=True)
+                sr_j = pd.concat([sr_j, new_serie], ignore_index=True)
             total += dcor.distance_correlation(sr_i, sr_j)
 
     total /= n * (n - 1) / 2

src/fairlens/metrics/distance.py

@@ -8,9 +8,8 @@
 
 import numpy as np
 import pandas as pd
-import pyemd
 from scipy.spatial.distance import jensenshannon
-from scipy.stats import entropy, kruskal, ks_2samp
+from scipy.stats import entropy, kruskal, ks_2samp, wasserstein_distance
 
 from .. import utils
 from ..metrics import significance as pv
@@ -304,19 +303,30 @@ class EarthMoversDistance(CategoricalDistanceMetric):
     """
 
     def distance_pdf(self, p: pd.Series, q: pd.Series, bin_edges: Optional[np.ndarray]) -> float:
-        distance_matrix = 1 - np.eye(len(p))
-
-        if bin_edges is not None:
-            # Use pair-wise euclidean distances between bin centers for scale data
-            bin_centers = np.mean([bin_edges[:-1], bin_edges[1:]], axis=0)
-            xx, yy = np.meshgrid(bin_centers, bin_centers)
-            distance_matrix = np.abs(xx - yy)
-
-        p = np.array(p).astype(np.float64)
-        q = np.array(q).astype(np.float64)
-        distance_matrix = distance_matrix.astype(np.float64)
+        p_sum = p.sum()
+        q_sum = q.sum()
+
+        if p_sum == 0 and q_sum == 0:
+            return 0.0
+        elif p_sum == 0 or q_sum == 0:
+            return 1.0
+
+        # normalise counts for consistency with scipy.stats.wasserstein
+        with np.errstate(divide="ignore", invalid="ignore"):
+            p_normalised = np.nan_to_num(p / p_sum).astype(np.float64)
+            q_normalised = np.nan_to_num(q / q_sum).astype(np.float64)
+
+        if bin_edges is None:
+            # if bins not given, histograms are assumed to be counts of nominal categories,
+            # and therefore distances betwen bins are meaningless. Set to all distances to
+            # unity to model this.
+            distance = 0.5 * np.sum(np.abs(p_normalised - q_normalised))
+        else:
+            # otherwise, use pair-wise euclidean distances between bin centers for scale data
+            bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
+            distance = wasserstein_distance(bin_centers, bin_centers, u_weights=p_normalised, v_weights=q_normalised)
 
-        return pyemd.emd(p, q, distance_matrix)
+        return distance
 
     @property
     def id(self) -> str:

src/fairlens/metrics/significance.py

@@ -14,7 +14,7 @@
 
 import numpy as np
 import pandas as pd
-from scipy.stats import beta, binom_test, norm
+from scipy.stats import beta, binomtest, norm
 
 
 def binominal_proportion_p_value(p_obs: float, p_null: float, n: int, alternative: str = "two-sided") -> float:
@@ -37,7 +37,8 @@ def binominal_proportion_p_value(p_obs: float, p_null: float, n: int, alternativ
     """
 
     k = np.ceil(p_obs * n)
-    return binom_test(k, n, p_null, alternative)
+    result = binomtest(k, n, p_null, alternative)
+    return result.pvalue
 
 
 def binominal_proportion_interval(

tests/test_metrics.py

@@ -45,7 +45,7 @@ def test_stat_distance_auto():
 
 def test_auto_binning():
     res = emd_samples(group1, group2)
-    assert stat_distance(df, target_attr, pred1, pred2, mode="emd")[0] == res
+    assert np.isclose(res, stat_distance(df, target_attr, pred1, pred2, mode="emd")[0], atol=1e-4)
 
 
 def test_mean_distance():