fpr fix (#84)

* fpr fix

* fix

.gitignore

@@ -26,3 +26,4 @@ site/
 .ipynb_checkpoints
 .ruff_cache
 __pycache__/
+.DS_Store

.pre-commit-config.yaml

@@ -1,5 +1,3 @@
-default_language_version:
-    python: python3.11
 repos:
 - repo: https://github.com/Lucas-C/pre-commit-hooks
   rev: v1.5.5

pyproject.toml

@@ -1,7 +1,7 @@
 [tool]
 [tool.poetry]
 name = "ep-stats"
-version = "2.5.0"
+version = "2.5.1"
 homepage = "https://github.com/avast/ep-stats"
 description = "Statistical package to evaluate ab tests in experimentation platform."
 authors = [

src/epstats/toolkit/experiment.py

@@ -773,7 +773,7 @@ def _get_false_positive_risk(self, metric_row: pd.Series) -> float:
         if self.null_hypothesis_rate is None:
             return np.nan
 
-        if metric_row["p_value"] >= metric_row["confidence_level"]:
+        if metric_row["p_value"] >= (1 - metric_row["confidence_level"]):
             return np.nan
 
         return Statistics.false_positive_risk(

src/epstats/toolkit/testing/resources/evaluations_metrics.csv

@@ -79,6 +79,6 @@ test-operator-precedence	b	1	Clicks	70	47	N/A	0.671429	N/A	-0.057644	N/A	N/A	N/A
 test-trigger	a	2	Average Bookings	1.0	10.0	N/A	10.0	N/A	0.0	N/A	N/A	N/A	N/A	N/A	N/A	N/A	N/A	N/A	N/A	N/A	0.95	N/A	1.0	N/A	N/A	N/A
 test-trigger	b	2	Average Bookings	2.0	23.0	N/A	11.5	N/A	0.15	0.7071067811865476	N/A	N/A	N/A	N/A	N/A	N/A	N/A	N/A	N/A	N/A	0.95	N/A	2.0	N/A	N/A	N/A
 test-false-positive-risk	a	1	Views per User of Screen button-1	1000	200	200	0.2	0.0	0.0	0.4002001501	0.0003203203	0.089487474	0.0	1998.0	1.0	1.0	1.0	0.95	N/A	0.17549854	0.95	N/A	1000	N/A	N/A	N/A
-test-false-positive-risk	b	1	Views per User of Screen button-1	1001	220	220	0.2197802198	0.0197802198	0.0989010989	0.4143046604	0.000331637	0.0955096411	1.0355090625	1996.7425963597	0.3005566728	0.3005566728	1.0	0.95	N/A	0.1873090388	0.95	0.05	1001	26043.0	0.079165	0.296686
+test-false-positive-risk	b	1	Views per User of Screen button-1	1001	220	220	0.2197802198	0.0197802198	0.0989010989	0.4143046604	0.000331637	0.0955096411	1.0355090625	1996.7425963597	0.3005566728	0.3005566728	1.0	0.95	N/A	0.1873090388	0.95	0.05	1001	26043.0	0.079165	NA
 test-false-positive-risk	a	2	Views per User of Screen button-%	1000	300	300	0.3	0.0	0.0	0.4584868703	0.0004204204	0.0683471872	0.0	1998.0	1.0	1.0	1.0	0.95	N/A	0.134039224	0.95	N/A	1000	N/A	N/A	N/A
 test-false-positive-risk	b	2	Views per User of Screen button-%	1001	400	400	0.3996003996	0.0996003996	0.332001332	0.4900610576	0.0004501301	0.0825214225	4.0232138728	1990.4598431179	5.95556e-05	5.95556e-05	1.0	0.95	N/A	0.1618374485	0.95	0.05	1001	15711.0	0.105139	0.000063

src/epstats/toolkit/testing/test_data.py

@@ -22,8 +22,8 @@ def load_goals_agg(cls, exp_id: str = None) -> pd.DataFrame:
         Arguments:
             exp_id: experiment id
         """
-        with files(resources) / "goals_agg.csv" as df_file:
-            goals_df = pd.read_csv(df_file, sep="\t")
+        df_file = files(resources).joinpath("goals_agg.csv")
+        goals_df = pd.read_csv(df_file, sep="\t")
         return goals_df[goals_df.exp_id == exp_id] if exp_id is not None else goals_df
 
     @classmethod
@@ -36,8 +36,8 @@ def load_goals_simple_agg(cls) -> pd.DataFrame:
 
         See `load_evaluations` set of functions to load corresponding evaluation results.
         """
-        with files(resources) / "goals_simple_agg.csv" as df_file:
-            goals_df = pd.read_csv(df_file, sep="\t")
+        df_file = files(resources).joinpath("goals_simple_agg.csv")
+        goals_df = pd.read_csv(df_file, sep="\t")
         return goals_df
 
     @classmethod
@@ -51,8 +51,8 @@ def load_goals_by_unit(cls, exp_id: str = None) -> pd.DataFrame:
         Arguments:
             exp_id: experiment id
         """
-        with files(resources) / "goals_by_unit.csv" as df_file:
-            goals_df = pd.read_csv(df_file, sep="\t")
+        df_file = files(resources).joinpath("goals_by_unit.csv")
+        goals_df = pd.read_csv(df_file, sep="\t")
         return goals_df[goals_df.exp_id == exp_id] if exp_id is not None else goals_df
 
     @classmethod
@@ -65,8 +65,8 @@ def load_evaluations_checks(cls, exp_id: str = None) -> pd.DataFrame:
         Arguments:
             exp_id: experiment id
         """
-        with files(resources) / "evaluations_checks.csv" as df_file:
-            goals_df = pd.read_csv(df_file, sep="\t")
+        df_file = files(resources).joinpath("evaluations_checks.csv")
+        goals_df = pd.read_csv(df_file, sep="\t")
         return goals_df[goals_df.exp_id == exp_id] if exp_id is not None else goals_df
 
     @classmethod
@@ -79,8 +79,8 @@ def load_evaluations_exposures(cls, exp_id: str = None) -> pd.DataFrame:
         Arguments:
             exp_id: experiment id
         """
-        with files(resources) / "evaluations_exposures.csv" as df_file:
-            exposures_df = pd.read_csv(df_file, sep="\t")
+        df_file = files(resources).joinpath("evaluations_exposures.csv")
+        exposures_df = pd.read_csv(df_file, sep="\t")
         return exposures_df[exposures_df.exp_id == exp_id] if exp_id is not None else exposures_df
 
     @classmethod
@@ -93,6 +93,6 @@ def load_evaluations_metrics(cls, exp_id: str = None) -> pd.DataFrame:
         Arguments:
             exp_id: experiment id
         """
-        with files(resources) / "evaluations_metrics.csv" as df_file:
-            goals_df = pd.read_csv(df_file, sep="\t")
+        df_file = files(resources).joinpath("evaluations_metrics.csv")
+        goals_df = pd.read_csv(df_file, sep="\t")
         return goals_df[goals_df.exp_id == exp_id] if exp_id is not None else goals_df