chi2 sum_ratio abs

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ep-stats"
-version = "2.5.7"
+version = "2.5.8"
 description = "Statistical package for controlled experimentation"
 readme = "README.md"
 authors = [

src/epstats/toolkit/check.py

@@ -31,7 +31,9 @@ def get_goals(self) -> List:
         """
         return self._goals
 
-    def evaluate_agg(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.DataFrame:
+    def evaluate_agg(
+        self, goals: pd.DataFrame, default_exp_variant_id: str
+    ) -> pd.DataFrame:
         """
         Evaluate this check from pre-aggregated goals.
 
@@ -56,7 +58,9 @@ def evaluate_agg(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.D
         """
         raise NotImplementedError()
 
-    def evaluate_by_unit(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.DataFrame:
+    def evaluate_by_unit(
+        self, goals: pd.DataFrame, default_exp_variant_id: str
+    ) -> pd.DataFrame:
         """
         Evaluate this check from goals aggregated by unit.
 
@@ -113,7 +117,9 @@ def __init__(
         super().__init__(id, name, denominator)
         self.confidence_level = confidence_level
 
-    def evaluate_agg(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.DataFrame:
+    def evaluate_agg(
+        self, goals: pd.DataFrame, default_exp_variant_id: str
+    ) -> pd.DataFrame:
         """
         See [`Check.evaluate_agg`][epstats.toolkit.check.Check.evaluate_agg].
         """
@@ -147,7 +153,9 @@ def evaluate_agg(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.D
         )
         return r
 
-    def evaluate_by_unit(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.DataFrame:
+    def evaluate_by_unit(
+        self, goals: pd.DataFrame, default_exp_variant_id: str
+    ) -> pd.DataFrame:
         """
         See [`Check.evaluate_by_unit`][epstats.toolkit.check.Check.evaluate_by_unit].
         """
@@ -247,7 +255,9 @@ def __init__(
         self._nominator_parser = Parser(nominator, nominator)
         self._goals = self._goals.union(self._nominator_parser.get_goals())
 
-    def evaluate_agg(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.DataFrame:
+    def evaluate_agg(
+        self, goals: pd.DataFrame, default_exp_variant_id: str
+    ) -> pd.DataFrame:
         """
         See [`Check.evaluate_agg`][epstats.toolkit.check.Check.evaluate_agg].
         """
@@ -257,18 +267,17 @@ def evaluate_agg(self, goals: pd.DataFrame, default_exp_variant_id: str) -> pd.D
 
         # chi-square test
         with np.errstate(divide="ignore", invalid="ignore"):
-            sum_ratio = nominator_counts.sum() / (denominator_counts.sum() + nominator_counts.sum())
+            sum_ratio = nominator_counts.sum() / (
+                denominator_counts.sum() + nominator_counts.sum()
+            )
 
             stat, pval = chisquare(
-                f_obs=[
-                    denominator_counts.sum() + 1,
-                    nominator_counts.sum() + 1,
-                ],
-                f_exp=[
-                    denominator_counts.sum() + nominator_counts.sum() + 1,
-                    1,
-                ],
+                [
+                    denominator_counts.sum(),
+                    np.abs(denominator_counts.sum() - nominator_counts.sum()),
+                ]
             )
+
         r = pd.DataFrame(
             {
                 "check_id": self.id,

src/epstats/toolkit/testing/resources/evaluations_checks.csv

@@ -61,13 +61,13 @@ test-multi-dimension,1,SRM,test_stat,0.00049975
 test-multi-dimension,1,SRM,confidence_level,0.999
 test-sum-ratio,1,EVA,sum_ratio,0.1639
 test-sum-ratio,1,EVA,max_sum_ratio,0.01
-test-sum-ratio,1,EVA,p_value,0.0
-test-sum-ratio,1,EVA,test_stat,9000491.7764
+test-sum-ratio,1,EVA,p_value,6.840197664885593e-73
+test-sum-ratio,1,EVA,test_stat,326.0869565217391
 test-sum-ratio,1,EVA,confidence_level,0.999
 test-multi-check,1,EVA,sum_ratio,0.1639
 test-multi-check,1,EVA,max_sum_ratio,0.01
-test-multi-check,1,EVA,p_value,0.0
-test-multi-check,1,EVA,test_stat,9000491.7764
+test-multi-check,1,EVA,p_value,6.840197664885593e-73
+test-multi-check,1,EVA,test_stat,326.0869565217391
 test-multi-check,1,EVA,confidence_level,0.999
 test-multi-check,2,SRM,p_value,0.140747987
 test-multi-check,2,SRM,test_stat,3.9215686