From 93a3596bf8ca5d4f5694c50e59d89890713a6aa1 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Tue, 24 Dec 2024 22:54:40 +0300 Subject: [PATCH 01/29] Redefine to_string without pandas --- src/tea_tasting/utils.py | 21 ++++++++++++++++++++- tests/test_utils.py | 10 ++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 97a6b43..514151f 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -288,7 +288,26 @@ def to_string( Look up for attributes `"{name}_lower"` and `"{name}_upper"`, and format the interval as `"[{lower_bound}, {lower_bound}]"`. """ - return self.to_pretty(keys, formatter).to_string(index=False) + if keys is None: + keys = self.default_keys + widths = {key: len(key) for key in keys} + + pretty_dicts = [] + for data in self.to_dicts(): + pretty_dict = {} + for key in keys: + val = formatter(data, key) + widths[key] = max(widths[key], len(val)) + pretty_dict |= {key: val} + pretty_dicts.append(pretty_dict) + + sep = " " + lines = [sep.join(key.rjust(widths[key]) for key in keys)] + lines.extend( + sep.join(pretty_dict[key].rjust(widths[key]) for key in keys) + for pretty_dict in pretty_dicts + ) + return "\n".join(lines) def to_html( self, diff --git a/tests/test_utils.py b/tests/test_utils.py index 3918602..8bce654 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +import textwrap from typing import TYPE_CHECKING import pandas as pd @@ -192,10 +193,11 @@ def test_pretty_dicts_mixin_to_pretty(pretty_dicts: tea_tasting.utils.PrettyDict ) def test_pretty_dicts_mixin_to_string(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert pretty_dicts.to_string() == pd.DataFrame({ - "a": ("0.123", "0.346", "0.568"), - "b": ("0.235", "0.457", "0.679"), - }).to_string(index=False) + assert pretty_dicts.to_string() == textwrap.dedent("""\ + a b + 0.123 0.235 + 0.346 0.457 + 0.568 0.679""") def test_pretty_dicts_mixin_to_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): assert pretty_dicts.to_html() == pd.DataFrame({ From c300fc6ca35e793dbd7071faca17ce0cdd5de456 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Tue, 24 Dec 2024 23:29:50 +0300 Subject: [PATCH 02/29] Redefine to_html without pandas --- src/tea_tasting/utils.py | 17 ++++++++++++++++- tests/test_experiment.py | 26 ++++++++------------------ tests/test_utils.py | 26 ++++++++++++++++++-------- 3 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 514151f..37cbad5 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -9,6 +9,7 @@ import locale import math from typing import TYPE_CHECKING +import xml.etree.ElementTree as ET import pandas as pd @@ -336,7 +337,21 @@ def to_html( Look up for attributes `"{name}_lower"` and `"{name}_upper"`, and format the interval as `"[{lower_bound}, {lower_bound}]"`. """ - return self.to_pretty(keys, formatter).to_html(index=False) + if keys is None: + keys = self.default_keys + table = ET.Element("table", {"class": "dataframe"}) + thead = ET.SubElement(table, "thead") + thead_tr = ET.SubElement(thead, "tr", {"style": "text-align: right;"}) + for key in keys: + th = ET.SubElement(thead_tr, "th") + th.text = key + tbody = ET.SubElement(table, "tbody") + for data in self.to_dicts(): + tr = ET.SubElement(tbody, "tr") + for key in keys: + td = ET.SubElement(tr, "td") + td.text = formatter(data, key) + return ET.tostring(table, encoding="unicode", method="html") def __str__(self) -> str: """Object string representation.""" diff --git a/tests/test_experiment.py b/tests/test_experiment.py index b313df9..01816d6 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -332,24 +332,14 @@ def test_experiment_result_to_string(result2: tea_tasting.experiment.ExperimentR )).to_string(index=False) def test_experiment_result_to_html(result2: tea_tasting.experiment.ExperimentResult): - assert result2.to_html() == pd.DataFrame(( - { - "metric": "metric_tuple", - "control": "4.44", - "treatment": "5.56", - "rel_effect_size": "20%", - "rel_effect_size_ci": "[12%, ∞]", - "pvalue": "0.235", - }, - { - "metric": "metric_dict", - "control": "10.0", - "treatment": "11.1", - "rel_effect_size": "11%", - "rel_effect_size_ci": "[0.0%, -]", - "pvalue": "-", - }, - )).to_html(index=False) + assert result2.to_html() == ( + '' + '' + '' + '' + '' + '
metriccontroltreatmentrel_effect_sizerel_effect_size_cipvalue
metric_tuple4.445.5620%[12%, ∞]0.235
metric_dict10.011.111%[0.0%, -]-
' + ) def test_experiment_results_to_dicts( diff --git a/tests/test_utils.py b/tests/test_utils.py index 8bce654..861a32e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -200,10 +200,15 @@ def test_pretty_dicts_mixin_to_string(pretty_dicts: tea_tasting.utils.PrettyDict 0.568 0.679""") def test_pretty_dicts_mixin_to_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert pretty_dicts.to_html() == pd.DataFrame({ - "a": ("0.123", "0.346", "0.568"), - "b": ("0.235", "0.457", "0.679"), - }).to_html(index=False) + assert pretty_dicts.to_html() == ( + '' + '' + '' + '' + '' + '' + '
ab
0.1230.235
0.3460.457
0.5680.679
' + ) def test_pretty_dicts_mixin_str(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): assert str(pretty_dicts) == pd.DataFrame({ @@ -212,10 +217,15 @@ def test_pretty_dicts_mixin_str(pretty_dicts: tea_tasting.utils.PrettyDictsMixin }).to_string(index=False) def test_pretty_dicts_mixin_repr_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert pretty_dicts._repr_html_() == pd.DataFrame({ - "a": ("0.123", "0.346", "0.568"), - "b": ("0.235", "0.457", "0.679"), - }).to_html(index=False) + assert pretty_dicts._repr_html_() == ( + '' + '' + '' + '' + '' + '' + '
ab
0.1230.235
0.3460.457
0.5680.679
' + ) def test_repr_mixin_repr(): From b8dfdb0825795b9e149e1782a30aec47d6ef92b9 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Tue, 24 Dec 2024 23:39:57 +0300 Subject: [PATCH 03/29] Fix to_string tests --- tests/test_experiment.py | 23 +++++------------------ tests/test_utils.py | 9 +++++---- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 01816d6..162d7f9 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -1,5 +1,6 @@ from __future__ import annotations +import textwrap from typing import TYPE_CHECKING, Any, NamedTuple, TypedDict import ibis @@ -312,24 +313,10 @@ def test_experiment_result_to_pretty(result2: tea_tasting.experiment.ExperimentR ) def test_experiment_result_to_string(result2: tea_tasting.experiment.ExperimentResult): - assert result2.to_string() == pd.DataFrame(( - { - "metric": "metric_tuple", - "control": "4.44", - "treatment": "5.56", - "rel_effect_size": "20%", - "rel_effect_size_ci": "[12%, ∞]", - "pvalue": "0.235", - }, - { - "metric": "metric_dict", - "control": "10.0", - "treatment": "11.1", - "rel_effect_size": "11%", - "rel_effect_size_ci": "[0.0%, -]", - "pvalue": "-", - }, - )).to_string(index=False) + assert result2.to_string() == textwrap.dedent("""\ + metric control treatment rel_effect_size rel_effect_size_ci pvalue + metric_tuple 4.44 5.56 20% [12%, ∞] 0.235 + metric_dict 10.0 11.1 11% [0.0%, -] -""") def test_experiment_result_to_html(result2: tea_tasting.experiment.ExperimentResult): assert result2.to_html() == ( diff --git a/tests/test_utils.py b/tests/test_utils.py index 861a32e..7b99943 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -211,10 +211,11 @@ def test_pretty_dicts_mixin_to_html(pretty_dicts: tea_tasting.utils.PrettyDictsM ) def test_pretty_dicts_mixin_str(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert str(pretty_dicts) == pd.DataFrame({ - "a": ("0.123", "0.346", "0.568"), - "b": ("0.235", "0.457", "0.679"), - }).to_string(index=False) + assert str(pretty_dicts) == textwrap.dedent("""\ + a b + 0.123 0.235 + 0.346 0.457 + 0.568 0.679""") def test_pretty_dicts_mixin_repr_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): assert pretty_dicts._repr_html_() == ( From fcbb6f774b6e68fd3373d66fd7738373b8b1009e Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Wed, 25 Dec 2024 23:02:17 +0300 Subject: [PATCH 04/29] Right alight html table --- src/tea_tasting/utils.py | 7 +++++-- tests/test_experiment.py | 2 +- tests/test_utils.py | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 37cbad5..4a78235 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -339,9 +339,12 @@ def to_html( """ if keys is None: keys = self.default_keys - table = ET.Element("table", {"class": "dataframe"}) + table = ET.Element( + "table", + {"class": "dataframe", "style": "text-align: right;"}, + ) thead = ET.SubElement(table, "thead") - thead_tr = ET.SubElement(thead, "tr", {"style": "text-align: right;"}) + thead_tr = ET.SubElement(thead, "tr") for key in keys: th = ET.SubElement(thead_tr, "th") th.text = key diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 162d7f9..0339bac 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -320,7 +320,7 @@ def test_experiment_result_to_string(result2: tea_tasting.experiment.ExperimentR def test_experiment_result_to_html(result2: tea_tasting.experiment.ExperimentResult): assert result2.to_html() == ( - '' + '
metric
' '' '' '' diff --git a/tests/test_utils.py b/tests/test_utils.py index 7b99943..45a469b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -201,8 +201,8 @@ def test_pretty_dicts_mixin_to_string(pretty_dicts: tea_tasting.utils.PrettyDict def test_pretty_dicts_mixin_to_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): assert pretty_dicts.to_html() == ( - '
metriccontroltreatmentrel_effect_sizerel_effect_size_cipvalue
metric_tuple4.445.5620%
' - '' + '
ab
' + '' '' '' '' @@ -219,8 +219,8 @@ def test_pretty_dicts_mixin_str(pretty_dicts: tea_tasting.utils.PrettyDictsMixin def test_pretty_dicts_mixin_repr_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): assert pretty_dicts._repr_html_() == ( - '
ab
0.1230.235
0.3460.457
' - '' + '
ab
' + '' '' '' '' From 0e56c4f68f5f4618e45972c084cd873a01bf79e1 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Wed, 25 Dec 2024 23:11:10 +0300 Subject: [PATCH 05/29] Optional indentation in to_html --- src/tea_tasting/experiment.py | 7 ++++++- src/tea_tasting/utils.py | 6 ++++++ tests/test_utils.py | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/tea_tasting/experiment.py b/src/tea_tasting/experiment.py index 0c74e1b..6e76fa2 100644 --- a/src/tea_tasting/experiment.py +++ b/src/tea_tasting/experiment.py @@ -236,6 +236,8 @@ def to_html( keys: Sequence[str] | None = None, formatter: Callable[ [dict[str, Any], str], str] = tea_tasting.utils.get_and_format_num, + *, + indent: str | None = None, ) -> str: """Convert the result to HTML. @@ -247,6 +249,8 @@ def to_html( formatter: Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value. + indent: Whitespace to insert for each indentation level. If `None`, + do not indent. Returns: A table with results rendered as HTML. @@ -306,7 +310,8 @@ def to_html( #>
ab
0.1230.235
0.3460.457
``` """ - return tea_tasting.utils.PrettyDictsMixin.to_html(self, keys, formatter) + return tea_tasting.utils.PrettyDictsMixin.to_html( + self, keys, formatter, indent=indent) class ExperimentResults( diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 4a78235..9cb7400 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -314,6 +314,8 @@ def to_html( self, keys: Sequence[str] | None = None, formatter: Callable[[dict[str, Any], str], str] = get_and_format_num, + *, + indent: str | None = None, ) -> str: """Convert the object to HTML. @@ -323,6 +325,8 @@ def to_html( formatter: Custom formatter function. It should accept a dictionary of metric result attributes and an attribute name, and return a formatted attribute value. + indent: Whitespace to insert for each indentation level. If `None`, + do not indent. Returns: A table with results rendered as HTML. @@ -354,6 +358,8 @@ def to_html( for key in keys: td = ET.SubElement(tr, "td") td.text = formatter(data, key) + if indent is not None: + ET.indent(table, space=indent) return ET.tostring(table, encoding="unicode", method="html") def __str__(self) -> str: diff --git a/tests/test_utils.py b/tests/test_utils.py index 45a469b..5c601e2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -210,6 +210,33 @@ def test_pretty_dicts_mixin_to_html(pretty_dicts: tea_tasting.utils.PrettyDictsM '' ) +def test_pretty_dicts_mixin_to_html_indent( + pretty_dicts: tea_tasting.utils.PrettyDictsMixin, +): + assert pretty_dicts.to_html(indent=" ") == textwrap.dedent("""\ + + + + + + + + + + + + + + + + + + + + + +
ab
0.1230.235
0.3460.457
0.5680.679
""") + def test_pretty_dicts_mixin_str(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): assert str(pretty_dicts) == textwrap.dedent("""\ a b From 81e6dc48a0b75857eba910e8c9aab6f4903c387a Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Wed, 25 Dec 2024 23:24:17 +0300 Subject: [PATCH 06/29] Make pandas optional in utils --- src/tea_tasting/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 9cb7400..6cf037e 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -11,13 +11,16 @@ from typing import TYPE_CHECKING import xml.etree.ElementTree as ET -import pandas as pd - if TYPE_CHECKING: from collections.abc import Callable, Iterator from typing import Any, Literal, TypeVar + try: + from pandas import DataFrame + except ImportError: + from typing import Any as DataFrame + R = TypeVar("R") @@ -224,15 +227,16 @@ class PrettyDictsMixin(abc.ABC): def to_dicts(self) -> Sequence[dict[str, Any]]: """Convert the object to a sequence of dictionaries.""" - def to_pandas(self) -> pd.DataFrame: + def to_pandas(self) -> DataFrame: """Convert the object to a Pandas DataFrame.""" + import pandas as pd return pd.DataFrame.from_records(self.to_dicts()) def to_pretty( self, keys: Sequence[str] | None = None, formatter: Callable[[dict[str, Any], str], str] = get_and_format_num, - ) -> pd.DataFrame: + ) -> DataFrame: """Convert the object to a Pandas Dataframe with formatted values. Args: @@ -255,6 +259,7 @@ def to_pretty( Look up for attributes `"{name}_lower"` and `"{name}_upper"`, and format the interval as `"[{lower_bound}, {lower_bound}]"`. """ + import pandas as pd if keys is None: keys = self.default_keys return pd.DataFrame.from_records( From d212fcf19bf7cf3614c3e5fa1cdde7facda75c35 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Thu, 26 Dec 2024 21:23:17 +0300 Subject: [PATCH 07/29] Config pyright --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7daf14a..f930725 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,4 +110,5 @@ reportMissingTypeStubs = false reportPrivateUsage = false reportUnknownArgumentType = false reportUnknownMemberType = false +reportUnknownParameterType = false reportUnknownVariableType = false From 0f1169d0b86e1414881d2dfe8830d7a27b3b909a Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Thu, 26 Dec 2024 21:23:58 +0300 Subject: [PATCH 08/29] Make synthetic data as PyArrow Table --- src/tea_tasting/datasets.py | 115 +++++++++++++++++++++++++++---- tests/metrics/test_base.py | 32 ++++++--- tests/metrics/test_mean.py | 18 ++--- tests/metrics/test_proportion.py | 12 ++-- tests/metrics/test_resampling.py | 11 +-- tests/test_aggr.py | 24 ++++--- tests/test_datasets.py | 107 +++++++++++++++++----------- tests/test_experiment.py | 24 ++++--- 8 files changed, 243 insertions(+), 100 deletions(-) diff --git a/src/tea_tasting/datasets.py b/src/tea_tasting/datasets.py index b585eb3..41517c1 100644 --- a/src/tea_tasting/datasets.py +++ b/src/tea_tasting/datasets.py @@ -3,19 +3,58 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, overload import numpy as np -import pandas as pd +import pyarrow as pa import tea_tasting.utils if TYPE_CHECKING: - from typing import Any + from typing import Any, Literal import numpy.typing as npt + try: + from pandas import DataFrame + except ImportError: + from typing import Any as DataFrame + + +@overload +def make_users_data( + *, + covariates: bool = False, + seed: int | np.random.Generator | np.random.SeedSequence | None = None, + n_users: int = 4000, + ratio: float | int = 1, + sessions_uplift: float | int = 0.0, + orders_uplift: float = 0.1, + revenue_uplift: float = 0.1, + avg_sessions: float | int = 2, + avg_orders_per_session: float = 0.25, + avg_revenue_per_order: float | int = 10, + to_pandas: Literal[False] = False, +) -> pa.Table: + ... + +@overload +def make_users_data( + *, + covariates: bool = False, + seed: int | np.random.Generator | np.random.SeedSequence | None = None, + n_users: int = 4000, + ratio: float | int = 1, + sessions_uplift: float | int = 0.0, + orders_uplift: float = 0.1, + revenue_uplift: float = 0.1, + avg_sessions: float | int = 2, + avg_orders_per_session: float = 0.25, + avg_revenue_per_order: float | int = 10, + to_pandas: Literal[True] = True, +) -> DataFrame: + ... def make_users_data( *, @@ -29,7 +68,8 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, -) -> pd.DataFrame: + to_pandas: bool = False, +) -> pa.Table | DataFrame: """Generate simulated data for A/B testing scenarios. Data mimics what you might encounter in an A/B test for an online store, @@ -63,6 +103,8 @@ def make_users_data( avg_orders_per_session: Average number of orders per session. Should be less than `1`. avg_revenue_per_order: Average revenue per order. + to_pandas: If set to `True`, returns a Pandas DataFrame; otherwise, + returns a PyArrow Table. Returns: Simulated data for A/B testing scenarios. @@ -122,10 +164,12 @@ def make_users_data( avg_sessions=avg_sessions, avg_orders_per_session=avg_orders_per_session, avg_revenue_per_order=avg_revenue_per_order, + to_pandas=to_pandas, explode_sessions=False, ) +@overload def make_sessions_data( *, covariates: bool = False, @@ -138,7 +182,41 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, -) -> pd.DataFrame: + to_pandas: Literal[False] = False, +) -> pa.Table: + ... + +@overload +def make_sessions_data( + *, + covariates: bool = False, + seed: int | np.random.Generator | np.random.SeedSequence | None = None, + n_users: int = 4000, + ratio: float | int = 1, + sessions_uplift: float | int = 0.0, + orders_uplift: float = 0.1, + revenue_uplift: float = 0.1, + avg_sessions: float | int = 2, + avg_orders_per_session: float = 0.25, + avg_revenue_per_order: float | int = 10, + to_pandas: Literal[True] = True, +) -> DataFrame: + ... + +def make_sessions_data( + *, + covariates: bool = False, + seed: int | np.random.Generator | np.random.SeedSequence | None = None, + n_users: int = 4000, + ratio: float | int = 1, + sessions_uplift: float | int = 0.0, + orders_uplift: float = 0.1, + revenue_uplift: float = 0.1, + avg_sessions: float | int = 2, + avg_orders_per_session: float = 0.25, + avg_revenue_per_order: float | int = 10, + to_pandas: bool = False, +) -> pa.Table | DataFrame: """Generate simulated user data for A/B testing scenarios. Data mimics what you might encounter in an A/B test for an online store, @@ -172,6 +250,8 @@ def make_sessions_data( avg_orders_per_session: Average number of orders per session. Should be less than `1`. avg_revenue_per_order: Average revenue per order. + to_pandas: If set to `True`, returns a Pandas DataFrame; otherwise, + returns a PyArrow Table. Returns: Simulated data for A/B testing scenarios. @@ -231,6 +311,7 @@ def make_sessions_data( avg_sessions=avg_sessions, avg_orders_per_session=avg_orders_per_session, avg_revenue_per_order=avg_revenue_per_order, + to_pandas=to_pandas, explode_sessions=True, ) @@ -248,7 +329,8 @@ def _make_data( avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, explode_sessions: bool = False, -) -> pd.DataFrame: + to_pandas: bool = False, +) -> pa.Table | DataFrame: _check_params( n_users=n_users, ratio=ratio, @@ -271,7 +353,7 @@ def _make_data( if explode_sessions: user = np.repeat(user, sessions) - sessions = 1 + sessions = np.ones_like(user) size = len(user) revenue_log_scale = np.sqrt(np.log( 1 + avg_sessions*(np.exp(revenue_log_scale**2) - 1))) @@ -299,13 +381,13 @@ def _make_data( revenue = orders * revenue_per_order - data = pd.DataFrame({ + data = { "user": user, "variant": variant[user].astype(np.uint8), "sessions": sessions, "orders": orders, "revenue": revenue, - }) + } if covariates: sessions_covariate = rng.poisson(lam=sessions / sessions_mult[user], size=size) @@ -333,13 +415,16 @@ def _make_data( orders_covariate = _avg_by_groups(orders_covariate, user) revenue_covariate = _avg_by_groups(revenue_covariate, user) - data = data.assign( - sessions_covariate=sessions_covariate, - orders_covariate=orders_covariate, - revenue_covariate=revenue_covariate, - ) + data |= { + "sessions_covariate": sessions_covariate, + "orders_covariate": orders_covariate, + "revenue_covariate": revenue_covariate, + } - return data + table = pa.table(data) + if to_pandas: + return table.to_pandas() + return table def _check_params( diff --git a/tests/metrics/test_base.py b/tests/metrics/test_base.py index 85be732..f49d192 100644 --- a/tests/metrics/test_base.py +++ b/tests/metrics/test_base.py @@ -6,6 +6,7 @@ import ibis import pandas as pd import polars as pl +import pyarrow as pa import pytest import tea_tasting.aggr @@ -65,28 +66,39 @@ def test_aggr_cols_len(): @pytest.fixture -def data_pandas() -> pd.DataFrame: - return tea_tasting.datasets.make_users_data(n_users=100, seed=42).astype( - {"variant": "int64"}) +def data_arrow() -> pa.Table: + table = tea_tasting.datasets.make_users_data(n_users=100, seed=42) + return table.set_column( + table.schema.get_field_index("variant"), + "variant", + table["variant"].cast(pa.int64()), + ) + +@pytest.fixture +def data_pandas(data_arrow: pa.Table) -> pd.DataFrame: + return data_arrow.to_pandas() @pytest.fixture -def data_polars(data_pandas: pd.DataFrame) -> pl.DataFrame: - return pl.from_pandas(data_pandas) +def data_polars(data_arrow: pa.Table) -> pl.DataFrame: + return pl.from_arrow(data_arrow) # type: ignore @pytest.fixture def data_polars_lazy(data_polars: pl.DataFrame) -> pl.LazyFrame: return data_polars.lazy() @pytest.fixture -def data_duckdb(data_pandas: pd.DataFrame) -> ibis.expr.types.Table: - return ibis.connect("duckdb://").create_table("data", data_pandas) +def data_duckdb(data_arrow: pa.Table) -> ibis.expr.types.Table: + return ibis.connect("duckdb://").create_table("data", data_arrow) @pytest.fixture -def data_sqlite(data_pandas: pd.DataFrame) -> ibis.expr.types.Table: - return ibis.connect("sqlite://").create_table("data", data_pandas) +def data_sqlite(data_arrow: pa.Table) -> ibis.expr.types.Table: + return ibis.connect("sqlite://").create_table("data", data_arrow) @pytest.fixture(params=[ - "data_pandas", "data_polars", "data_polars_lazy", "data_duckdb", "data_sqlite"]) + "data_arrow", "data_pandas", + "data_polars", "data_polars_lazy", + "data_duckdb", "data_sqlite", +]) def data(request: pytest.FixtureRequest) -> Frame: return request.getfixturevalue(request.param) diff --git a/tests/metrics/test_mean.py b/tests/metrics/test_mean.py index 0c6c21e..dabc6cc 100644 --- a/tests/metrics/test_mean.py +++ b/tests/metrics/test_mean.py @@ -15,21 +15,21 @@ if TYPE_CHECKING: from typing import Any - import pandas as pd + import pyarrow as pa @pytest.fixture -def data_pandas() -> pd.DataFrame: +def data_arrow() -> pa.Table: return tea_tasting.datasets.make_users_data(n_users=100, covariates=True, seed=42) @pytest.fixture -def data_aggr(data_pandas: pd.DataFrame) -> dict[Any, tea_tasting.aggr.Aggregates]: +def data_aggr(data_arrow: pa.Table) -> dict[Any, tea_tasting.aggr.Aggregates]: cols = ( "sessions", "orders", "revenue", "sessions_covariate", "orders_covariate", "revenue_covariate", ) return tea_tasting.aggr.read_aggregates( - data_pandas, + data_arrow, group_col="variant", has_count=True, mean_cols=cols, @@ -43,14 +43,14 @@ def data_aggr(data_pandas: pd.DataFrame) -> dict[Any, tea_tasting.aggr.Aggregate ) @pytest.fixture -def power_data_pandas() -> pd.DataFrame: +def power_data_pandas() -> pa.Table: return tea_tasting.datasets.make_users_data( n_users=100, covariates=True, seed=42, sessions_uplift=0, orders_uplift=0, revenue_uplift=0, ) @pytest.fixture -def power_data_aggr(power_data_pandas: pd.DataFrame) -> tea_tasting.aggr.Aggregates: +def power_data_aggr(power_data_pandas: pa.Table) -> tea_tasting.aggr.Aggregates: cols = ( "sessions", "orders", "revenue", "sessions_covariate", "orders_covariate", "revenue_covariate", @@ -157,12 +157,12 @@ def test_ratio_of_means_aggr_cols(): assert set(aggr_cols.cov_cols) == {("a", "b"), ("a", "c"), ("b", "c")} -def test_ratio_of_means_analyze_frame(data_pandas: pd.DataFrame): +def test_ratio_of_means_analyze_frame(data_arrow: pa.Table): metric = tea_tasting.metrics.mean.RatioOfMeans( numer="orders", denom="sessions", ) - result = metric.analyze(data_pandas, 0, 1, variant="variant") + result = metric.analyze(data_arrow, 0, 1, variant="variant") assert isinstance(result, tea_tasting.metrics.mean.MeanResult) def test_ratio_of_means_analyze_basic( @@ -229,7 +229,7 @@ def test_ratio_of_means_analyze_ratio_less_use_norm( assert result.statistic == pytest.approx(-0.3573188986307722) -def test_ratio_of_means_solve_power_frame(power_data_pandas: pd.DataFrame): +def test_ratio_of_means_solve_power_frame(power_data_pandas: pa.Table): metric = tea_tasting.metrics.mean.RatioOfMeans( numer="orders", denom="sessions", diff --git a/tests/metrics/test_proportion.py b/tests/metrics/test_proportion.py index ae3fbc4..95e0be9 100644 --- a/tests/metrics/test_proportion.py +++ b/tests/metrics/test_proportion.py @@ -14,17 +14,17 @@ if TYPE_CHECKING: from typing import Any - import pandas as pd + import pyarrow as pa @pytest.fixture -def data_pandas() -> pd.DataFrame: +def data_arrow() -> pa.Table: return tea_tasting.datasets.make_users_data(n_users=100, seed=42) @pytest.fixture -def data_aggr(data_pandas: pd.DataFrame) -> dict[Any, tea_tasting.aggr.Aggregates]: +def data_aggr(data_arrow: pa.Table) -> dict[Any, tea_tasting.aggr.Aggregates]: return tea_tasting.aggr.read_aggregates( - data_pandas, + data_arrow, group_col="variant", has_count=True, mean_cols=(), @@ -55,9 +55,9 @@ def test_sample_ratio_aggr_cols(): assert metric.aggr_cols == tea_tasting.metrics.base.AggrCols(has_count=True) -def test_sample_ratio_analyze_frame(data_pandas: pd.DataFrame): +def test_sample_ratio_analyze_frame(data_arrow: pa.Table): metric = tea_tasting.metrics.proportion.SampleRatio() - result = metric.analyze(data_pandas, 0, 1, variant="variant") + result = metric.analyze(data_arrow, 0, 1, variant="variant") assert isinstance(result, tea_tasting.metrics.proportion.SampleRatioResult) def test_sample_ratio_analyze_auto(): diff --git a/tests/metrics/test_resampling.py b/tests/metrics/test_resampling.py index bf4b267..02ad643 100644 --- a/tests/metrics/test_resampling.py +++ b/tests/metrics/test_resampling.py @@ -17,16 +17,17 @@ import numpy.typing as npt import pandas as pd + import pyarrow as pa @pytest.fixture -def data_pandas() -> pd.DataFrame: +def data_arrow() -> pd.DataFrame: return tea_tasting.datasets.make_users_data(n_users=100, seed=42) @pytest.fixture -def data_gran(data_pandas: pd.DataFrame) -> dict[Any, pd.DataFrame]: +def data_gran(data_arrow: pa.Table) -> dict[Any, pd.DataFrame]: return tea_tasting.metrics.base.read_dataframes( - data_pandas, + data_arrow, ("sessions", "orders", "revenue"), variant="variant", ) @@ -72,9 +73,9 @@ def test_bootstrap_cols(): assert metric.cols == ("a", "b") -def test_bootstrap_analyze_frame(data_pandas: pd.DataFrame): +def test_bootstrap_analyze_frame(data_arrow: pa.Table): metric = tea_tasting.metrics.resampling.Bootstrap("sessions", np.mean) - result = metric.analyze(data_pandas, 0, 1, variant="variant") + result = metric.analyze(data_arrow, 0, 1, variant="variant") assert isinstance(result, tea_tasting.metrics.resampling.BootstrapResult) diff --git a/tests/test_aggr.py b/tests/test_aggr.py index fc10854..7a07c1e 100644 --- a/tests/test_aggr.py +++ b/tests/test_aggr.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: import ibis.expr.types # noqa: TC004 import pandas as pd + import pyarrow as pa Frame = ibis.expr.types.Table | pd.DataFrame | pl.LazyFrame @@ -34,27 +35,34 @@ def aggr() -> tea_tasting.aggr.Aggregates: @pytest.fixture -def data_pandas() -> pd.DataFrame: +def data_arrow() -> pa.Table: return tea_tasting.datasets.make_users_data(n_users=100, seed=42) @pytest.fixture -def data_polars(data_pandas: pd.DataFrame) -> pl.DataFrame: - return pl.from_pandas(data_pandas) +def data_pandas(data_arrow: pa.Table) -> pd.DataFrame: + return data_arrow.to_pandas() + +@pytest.fixture +def data_polars(data_arrow: pa.Table) -> pl.DataFrame: + return pl.from_arrow(data_arrow) # type: ignore @pytest.fixture def data_polars_lazy(data_polars: pl.DataFrame) -> pl.LazyFrame: return data_polars.lazy() @pytest.fixture -def data_duckdb(data_pandas: pd.DataFrame) -> ibis.expr.types.Table: - return ibis.connect("duckdb://").create_table("data", data_pandas) +def data_duckdb(data_arrow: pa.Table) -> ibis.expr.types.Table: + return ibis.connect("duckdb://").create_table("data", data_arrow) @pytest.fixture -def data_sqlite(data_pandas: pd.DataFrame) -> ibis.expr.types.Table: - return ibis.connect("sqlite://").create_table("data", data_pandas) +def data_sqlite(data_arrow: pa.Table) -> ibis.expr.types.Table: + return ibis.connect("sqlite://").create_table("data", data_arrow) @pytest.fixture(params=[ - "data_pandas", "data_polars", "data_polars_lazy", "data_duckdb", "data_sqlite"]) + "data_arrow", "data_pandas", + "data_polars", "data_polars_lazy", + "data_duckdb", "data_sqlite", +]) def data(request: pytest.FixtureRequest) -> Frame: return request.getfixturevalue(request.param) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index a6fc6cc..657bde2 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -1,6 +1,9 @@ +# pyright: reportAttributeAccessIssue=false from __future__ import annotations import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc import tea_tasting.datasets @@ -8,71 +11,97 @@ def test_make_users_data_default(): n_users = 100 data = tea_tasting.datasets.make_users_data(seed=42, n_users=n_users) + assert isinstance(data, pa.Table) + assert data.column_names == ["user", "variant", "sessions", "orders", "revenue"] + assert data.num_rows == n_users + assert pc.count_distinct(data["user"]).as_py() == n_users + assert pc.count_distinct(data["variant"]).as_py() == 2 + assert pc.min(data["sessions"]).as_py() > 0 + assert pc.min(data["orders"]).as_py() >= 0 + assert pc.min(data["revenue"]).as_py() >= 0 + assert pc.min(pc.subtract(data["orders"], data["sessions"])).as_py() <= 0 + assert int(pc.min(pc.equal( + pc.greater_equal(data["revenue"], 0), + pc.greater_equal(data["orders"], 0), + )).as_py()) == 1 + +def test_make_users_data_pandas(): + n_users = 100 + data = tea_tasting.datasets.make_users_data( + seed=42, n_users=n_users, to_pandas=True) assert isinstance(data, pd.DataFrame) assert data.columns.to_list() == [ "user", "variant", "sessions", "orders", "revenue"] - assert len(data) == n_users - assert data["user"].drop_duplicates().count() == n_users - assert data["variant"].drop_duplicates().count() == 2 - assert data["sessions"].min() > 0 - assert data["orders"].min() >= 0 - assert data["orders"].sub(data["sessions"]).min() <= 0 - assert data["revenue"].min() >= 0 - assert data["revenue"].gt(0).eq(data["orders"].gt(0)).astype(int).min() == 1 + assert data.shape[0] == n_users def test_make_users_data_covariates(): n_users = 100 data = tea_tasting.datasets.make_users_data( seed=42, covariates=True, n_users=n_users) - assert isinstance(data, pd.DataFrame) - assert data.columns.to_list() == [ + assert isinstance(data, pa.Table) + assert data.column_names == [ "user", "variant", "sessions", "orders", "revenue", "sessions_covariate", "orders_covariate", "revenue_covariate", ] - assert data["sessions_covariate"].min() >= 0 - assert data["orders_covariate"].min() >= 0 - assert data["orders_covariate"].sub(data["sessions_covariate"]).min() <= 0 - assert data["revenue_covariate"].min() >= 0 - assert ( - data["revenue_covariate"].gt(0) - .eq(data["orders_covariate"].gt(0)) - .astype(int).min() - ) == 1 + assert pc.min(data["sessions_covariate"]).as_py() >= 0 + assert pc.min(data["orders_covariate"]).as_py() >= 0 + assert pc.min(data["revenue_covariate"]).as_py() >= 0 + assert pc.min(pc.subtract( + data["orders_covariate"], + data["sessions_covariate"], + )).as_py() <= 0 + assert int(pc.min(pc.equal( + pc.greater_equal(data["revenue_covariate"], 0), + pc.greater_equal(data["orders_covariate"], 0), + )).as_py()) == 1 def test_make_sessions_data_default(): n_users = 100 data = tea_tasting.datasets.make_sessions_data(seed=42, n_users=n_users) + assert isinstance(data, pa.Table) + assert data.column_names == ["user", "variant", "sessions", "orders", "revenue"] + assert data.num_rows > n_users + assert pc.count_distinct(data["user"]).as_py() == n_users + assert pc.count_distinct(data["variant"]).as_py() == 2 + assert pc.min(data["sessions"]).as_py() == 1 + assert pc.max(data["sessions"]).as_py() == 1 + assert pc.min(data["orders"]).as_py() >= 0 + assert pc.min(data["revenue"]).as_py() >= 0 + assert pc.min(pc.subtract(data["orders"], data["sessions"])).as_py() <= 0 + assert int(pc.min(pc.equal( + pc.greater_equal(data["revenue"], 0), + pc.greater_equal(data["orders"], 0), + )).as_py()) == 1 + +def test_make_sessions_data_pandas(): + n_users = 100 + data = tea_tasting.datasets.make_sessions_data( + seed=42, n_users=n_users, to_pandas=True) assert isinstance(data, pd.DataFrame) assert data.columns.to_list() == [ "user", "variant", "sessions", "orders", "revenue"] - assert len(data) > n_users - assert data["user"].drop_duplicates().count() == n_users - assert data["variant"].drop_duplicates().count() == 2 - assert data["sessions"].min() == 1 - assert data["sessions"].max() == 1 - assert data["orders"].min() >= 0 - assert data["orders"].sub(data["sessions"]).min() <= 0 - assert data["revenue"].min() >= 0 - assert data["revenue"].gt(0).eq(data["orders"].gt(0)).astype(int).min() == 1 + assert data.shape[0] > n_users def test_make_sessions_data_covariates(): n_users = 100 data = tea_tasting.datasets.make_sessions_data( seed=42, covariates=True, n_users=n_users) - assert isinstance(data, pd.DataFrame) - assert data.columns.to_list() == [ + assert isinstance(data, pa.Table) + assert data.column_names == [ "user", "variant", "sessions", "orders", "revenue", "sessions_covariate", "orders_covariate", "revenue_covariate", ] - assert data["sessions_covariate"].min() >= 0 - assert data["orders_covariate"].min() >= 0 - assert data["orders_covariate"].sub(data["sessions_covariate"]).min() <= 0 - assert data["revenue_covariate"].min() >= 0 - assert ( - data["revenue_covariate"].gt(0) - .eq(data["orders_covariate"].gt(0)) - .astype(int).min() - ) == 1 + assert pc.min(data["sessions_covariate"]).as_py() >= 0 + assert pc.min(data["orders_covariate"]).as_py() >= 0 + assert pc.min(data["revenue_covariate"]).as_py() >= 0 + assert pc.min(pc.subtract( + data["orders_covariate"], + data["sessions_covariate"], + )).as_py() <= 0 + assert int(pc.min(pc.equal( + pc.greater_equal(data["revenue_covariate"], 0), + pc.greater_equal(data["orders_covariate"], 0), + )).as_py()) == 1 diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 0339bac..549eef6 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -22,6 +22,7 @@ from typing import Literal import narwhals.typing # noqa: TC004 + import pyarrow as pa Frame = ibis.expr.types.Table | pd.DataFrame | pl.LazyFrame @@ -226,27 +227,34 @@ def results2( @pytest.fixture -def data_pandas() -> pd.DataFrame: +def data_arrow() -> pa.Table: return tea_tasting.datasets.make_users_data(n_users=100, seed=42) @pytest.fixture -def data_polars(data_pandas: pd.DataFrame) -> pl.DataFrame: - return pl.from_pandas(data_pandas) +def data_pandas(data_arrow: pa.Table) -> pd.DataFrame: + return data_arrow.to_pandas() + +@pytest.fixture +def data_polars(data_arrow: pa.Table) -> pl.DataFrame: + return pl.from_arrow(data_arrow) # type: ignore @pytest.fixture def data_polars_lazy(data_polars: pl.DataFrame) -> pl.LazyFrame: return data_polars.lazy() @pytest.fixture -def data_duckdb(data_pandas: pd.DataFrame) -> ibis.expr.types.Table: - return ibis.connect("duckdb://").create_table("data", data_pandas) +def data_duckdb(data_arrow: pa.Table) -> ibis.expr.types.Table: + return ibis.connect("duckdb://").create_table("data", data_arrow) @pytest.fixture -def data_sqlite(data_pandas: pd.DataFrame) -> ibis.expr.types.Table: - return ibis.connect("sqlite://").create_table("data", data_pandas) +def data_sqlite(data_arrow: pa.Table) -> ibis.expr.types.Table: + return ibis.connect("sqlite://").create_table("data", data_arrow) @pytest.fixture(params=[ - "data_pandas", "data_polars", "data_polars_lazy", "data_duckdb", "data_sqlite"]) + "data_arrow", "data_pandas", + "data_polars", "data_polars_lazy", + "data_duckdb", "data_sqlite", +]) def data(request: pytest.FixtureRequest) -> Frame: return request.getfixturevalue(request.param) From faa1a059266f2d7a944900af5dbe40b0e0c780b3 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Thu, 26 Dec 2024 21:32:09 +0300 Subject: [PATCH 09/29] Minor change --- src/tea_tasting/datasets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tea_tasting/datasets.py b/src/tea_tasting/datasets.py index 41517c1..62d5853 100644 --- a/src/tea_tasting/datasets.py +++ b/src/tea_tasting/datasets.py @@ -421,10 +421,10 @@ def _make_data( "revenue_covariate": revenue_covariate, } - table = pa.table(data) if to_pandas: - return table.to_pandas() - return table + import pandas as pd + return pd.DataFrame(data) + return pa.table(data) def _check_params( From 3b35e547c9672f6d01b5db45b06d6919cb10aa5f Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 00:14:57 +0300 Subject: [PATCH 10/29] Minor fix --- tests/metrics/test_base.py | 2 +- tests/test_aggr.py | 2 +- tests/test_experiment.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/metrics/test_base.py b/tests/metrics/test_base.py index f49d192..940d224 100644 --- a/tests/metrics/test_base.py +++ b/tests/metrics/test_base.py @@ -20,7 +20,7 @@ import ibis.expr.types # noqa: TC004 - Frame = ibis.expr.types.Table | pd.DataFrame | pl.LazyFrame + Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame def test_aggr_cols_or(): diff --git a/tests/test_aggr.py b/tests/test_aggr.py index 7a07c1e..c2157e5 100644 --- a/tests/test_aggr.py +++ b/tests/test_aggr.py @@ -16,7 +16,7 @@ import pyarrow as pa - Frame = ibis.expr.types.Table | pd.DataFrame | pl.LazyFrame + Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame COUNT = 100 diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 549eef6..7488ce7 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -25,7 +25,7 @@ import pyarrow as pa - Frame = ibis.expr.types.Table | pd.DataFrame | pl.LazyFrame + Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame class _MetricResultTuple(NamedTuple): From 7b8b86f3201a0490dc1aa388621cc0ba851e7adc Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 00:45:19 +0300 Subject: [PATCH 11/29] Minor change --- src/tea_tasting/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 6cf037e..0d55db0 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -308,12 +308,12 @@ def to_string( pretty_dicts.append(pretty_dict) sep = " " - lines = [sep.join(key.rjust(widths[key]) for key in keys)] - lines.extend( + rows = [sep.join(key.rjust(widths[key]) for key in keys)] + rows.extend( sep.join(pretty_dict[key].rjust(widths[key]) for key in keys) for pretty_dict in pretty_dicts ) - return "\n".join(lines) + return "\n".join(rows) def to_html( self, From 1206728669b848d98339d0c260b013ebbbfef2f6 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 22:23:32 +0300 Subject: [PATCH 12/29] Remove unnecessary methods --- src/tea_tasting/experiment.py | 232 ---------------------------------- tests/test_experiment.py | 51 -------- 2 files changed, 283 deletions(-) diff --git a/src/tea_tasting/experiment.py b/src/tea_tasting/experiment.py index 6e76fa2..7578cd5 100644 --- a/src/tea_tasting/experiment.py +++ b/src/tea_tasting/experiment.py @@ -15,7 +15,6 @@ if TYPE_CHECKING: - from collections.abc import Callable, Sequence from typing import Literal import narwhals.typing # noqa: TC004 @@ -82,237 +81,6 @@ def to_dicts(self) -> tuple[dict[str, Any], ...]: for k, v in self.items() ) - def to_pandas(self) -> pd.DataFrame: - """Convert the result to a Pandas DataFrame. - - Examples: - ```python - import tea_tasting as tt - - - experiment = tt.Experiment( - sessions_per_user=tt.Mean("sessions"), - orders_per_session=tt.RatioOfMeans("orders", "sessions"), - orders_per_user=tt.Mean("orders"), - revenue_per_user=tt.Mean("revenue"), - ) - - data = tt.make_users_data(seed=42) - result = experiment.analyze(data) - print(result.to_pandas()) - #> metric control ... pvalue statistic - #> 0 sessions_per_user 1.996045 ... 0.674021 -0.420667 - #> 1 orders_per_session 0.265726 ... 0.076238 1.773406 - #> 2 orders_per_user 0.530400 ... 0.117732 1.564703 - #> 3 revenue_per_user 5.241079 ... 0.123097 1.542231 - #> - #> [4 rows x 11 columns] - ``` - """ - return tea_tasting.utils.PrettyDictsMixin.to_pandas(self) - - def to_pretty( - self, - keys: Sequence[str] | None = None, - formatter: Callable[ - [dict[str, Any], str], str] = tea_tasting.utils.get_and_format_num, - ) -> pd.DataFrame: - """Convert the result to a Pandas Dataframe with formatted values. - - Metric result attribute values are converted to strings in a "pretty" format. - - Args: - keys: Metric attribute names. If an attribute is not defined - for a metric it's assumed to be `None`. - formatter: Custom formatter function. It should accept a dictionary - of metric result attributes and an attribute name, and return - a formatted attribute value. - - Returns: - Pandas Dataframe with formatted values. - - Default formatting rules: - - If a name starts with `"rel_"` consider it a percentage value. - Round percentage values to 2 significant digits, multiply by `100` - and add `"%"`. - - Round other values to 3 significant values. - - If value is less than `0.001`, format it in exponential presentation. - - If a name ends with `"_ci"`, consider it a confidence interval. - Look up for attributes `"{name}_lower"` and `"{name}_upper"`, - and format the interval as `"[{lower_bound}, {lower_bound}]"`. - - Examples: - ```python - import tea_tasting as tt - - - experiment = tt.Experiment( - sessions_per_user=tt.Mean("sessions"), - orders_per_session=tt.RatioOfMeans("orders", "sessions"), - orders_per_user=tt.Mean("orders"), - revenue_per_user=tt.Mean("revenue"), - ) - - data = tt.make_users_data(seed=42) - result = experiment.analyze(data) - print(result.to_pretty(keys=( - "metric", - "control", - "treatment", - "effect_size", - "effect_size_ci", - ))) - #> metric control treatment effect_size effect_size_ci - #> 0 sessions_per_user 2.00 1.98 -0.0132 [-0.0750, 0.0485] - #> 1 orders_per_session 0.266 0.289 0.0233 [-0.00246, 0.0491] - #> 2 orders_per_user 0.530 0.573 0.0427 [-0.0108, 0.0962] - #> 3 revenue_per_user 5.24 5.73 0.489 [-0.133, 1.11] - ``` - """ - return tea_tasting.utils.PrettyDictsMixin.to_pretty(self, keys, formatter) - - def to_string( - self, - keys: Sequence[str] | None = None, - formatter: Callable[ - [dict[str, Any], str], str] = tea_tasting.utils.get_and_format_num, - ) -> str: - """Convert the result to a string. - - Metric result attribute values are converted to strings in a "pretty" format. - - Args: - keys: Metric attribute names. If an attribute is not defined - for a metric it's assumed to be `None`. - formatter: Custom formatter function. It should accept a dictionary - of metric result attributes and an attribute name, and return - a formatted attribute value. - - Returns: - A string with formatted values. - - Default formatting rules: - - If a name starts with `"rel_"` consider it a percentage value. - Round percentage values to 2 significant digits, multiply by `100` - and add `"%"`. - - Round other values to 3 significant values. - - If value is less than `0.001`, format it in exponential presentation. - - If a name ends with `"_ci"`, consider it a confidence interval. - Look up for attributes `"{name}_lower"` and `"{name}_upper"`, - and format the interval as `"[{lower_bound}, {lower_bound}]"`. - - Examples: - ```python - import tea_tasting as tt - - - experiment = tt.Experiment( - sessions_per_user=tt.Mean("sessions"), - orders_per_session=tt.RatioOfMeans("orders", "sessions"), - orders_per_user=tt.Mean("orders"), - revenue_per_user=tt.Mean("revenue"), - ) - - data = tt.make_users_data(seed=42) - result = experiment.analyze(data) - print(result.to_string(keys=( - "metric", - "control", - "treatment", - "effect_size", - "effect_size_ci", - ))) - #> metric control treatment effect_size effect_size_ci - #> sessions_per_user 2.00 1.98 -0.0132 [-0.0750, 0.0485] - #> orders_per_session 0.266 0.289 0.0233 [-0.00246, 0.0491] - #> orders_per_user 0.530 0.573 0.0427 [-0.0108, 0.0962] - #> revenue_per_user 5.24 5.73 0.489 [-0.133, 1.11] - ``` - """ - return tea_tasting.utils.PrettyDictsMixin.to_string(self, keys, formatter) - - def to_html( - self, - keys: Sequence[str] | None = None, - formatter: Callable[ - [dict[str, Any], str], str] = tea_tasting.utils.get_and_format_num, - *, - indent: str | None = None, - ) -> str: - """Convert the result to HTML. - - Metric result attribute values are converted to strings in a "pretty" format. - - Args: - keys: Metric attribute names. If an attribute is not defined - for a metric it's assumed to be `None`. - formatter: Custom formatter function. It should accept a dictionary - of metric result attributes and an attribute name, and return - a formatted attribute value. - indent: Whitespace to insert for each indentation level. If `None`, - do not indent. - - Returns: - A table with results rendered as HTML. - - Default formatting rules: - - If a name starts with `"rel_"` consider it a percentage value. - Round percentage values to 2 significant digits, multiply by `100` - and add `"%"`. - - Round other values to 3 significant values. - - If value is less than `0.001`, format it in exponential presentation. - - If a name ends with `"_ci"`, consider it a confidence interval. - Look up for attributes `"{name}_lower"` and `"{name}_upper"`, - and format the interval as `"[{lower_bound}, {lower_bound}]"`. - - Examples: - ```python - import tea_tasting as tt - - - experiment = tt.Experiment( - orders_per_user=tt.Mean("orders"), - revenue_per_user=tt.Mean("revenue"), - ) - - data = tt.make_users_data(seed=42) - result = experiment.analyze(data) - print(result.to_html()) - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #> - #>
metriccontroltreatmentrel_effect_sizerel_effect_size_cipvalue
orders_per_user0.5300.5738.0%[-2.0%, 19%]0.118
revenue_per_user5.245.739.3%[-2.4%, 22%]0.123
- ``` - """ - return tea_tasting.utils.PrettyDictsMixin.to_html( - self, keys, formatter, indent=indent) - class ExperimentResults( UserDict[tuple[Any, Any], ExperimentResult], diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 7488ce7..64266cf 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -1,6 +1,5 @@ from __future__ import annotations -import textwrap from typing import TYPE_CHECKING, Any, NamedTuple, TypedDict import ibis @@ -286,56 +285,6 @@ def test_experiment_result_to_dicts(result: tea_tasting.experiment.ExperimentRes {"metric": "metric_dict", "control": 20, "treatment": 22, "effect_size": 2}, ) -def test_experiment_result_to_pandas(result: tea_tasting.experiment.ExperimentResult): - pd.testing.assert_frame_equal( - result.to_pandas(), - pd.DataFrame({ - "metric": ("metric_tuple", "metric_dict"), - "control": (10, 20), - "treatment": (11, 22), - "effect_size": (1, 2), - }), - ) - -def test_experiment_result_to_pretty(result2: tea_tasting.experiment.ExperimentResult): - pd.testing.assert_frame_equal( - result2.to_pretty(), - pd.DataFrame(( - { - "metric": "metric_tuple", - "control": "4.44", - "treatment": "5.56", - "rel_effect_size": "20%", - "rel_effect_size_ci": "[12%, ∞]", - "pvalue": "0.235", - }, - { - "metric": "metric_dict", - "control": "10.0", - "treatment": "11.1", - "rel_effect_size": "11%", - "rel_effect_size_ci": "[0.0%, -]", - "pvalue": "-", - }, - )), - ) - -def test_experiment_result_to_string(result2: tea_tasting.experiment.ExperimentResult): - assert result2.to_string() == textwrap.dedent("""\ - metric control treatment rel_effect_size rel_effect_size_ci pvalue - metric_tuple 4.44 5.56 20% [12%, ∞] 0.235 - metric_dict 10.0 11.1 11% [0.0%, -] -""") - -def test_experiment_result_to_html(result2: tea_tasting.experiment.ExperimentResult): - assert result2.to_html() == ( - '' - '' - '' - '' - '' - '
metriccontroltreatmentrel_effect_sizerel_effect_size_cipvalue
metric_tuple4.445.5620%[12%, ∞]0.235
metric_dict10.011.111%[0.0%, -]-
' - ) - def test_experiment_results_to_dicts( results2: tea_tasting.experiment.ExperimentResults, From ec2393aafaaa91f8f40356448115cbcd226c6858 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 22:29:50 +0300 Subject: [PATCH 13/29] Rename to_pretty -> to_pretty_dicts and change result type --- src/tea_tasting/utils.py | 14 +++++--------- tests/test_utils.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 0d55db0..630c449 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -232,12 +232,12 @@ def to_pandas(self) -> DataFrame: import pandas as pd return pd.DataFrame.from_records(self.to_dicts()) - def to_pretty( + def to_pretty_dicts( self, keys: Sequence[str] | None = None, formatter: Callable[[dict[str, Any], str], str] = get_and_format_num, - ) -> DataFrame: - """Convert the object to a Pandas Dataframe with formatted values. + ) -> list[dict[str, Any]]: + """Convert the object to a list of dictionaries with formatted values. Args: keys: Keys to convert. If a key is not defined in the dictionary @@ -247,7 +247,7 @@ def to_pretty( a formatted attribute value. Returns: - Pandas Dataframe with formatted values. + List of dictionaries with formatted values. Default formatting rules: - If a name starts with `"rel_"` or equals to `"power"` consider it @@ -259,13 +259,9 @@ def to_pretty( Look up for attributes `"{name}_lower"` and `"{name}_upper"`, and format the interval as `"[{lower_bound}, {lower_bound}]"`. """ - import pandas as pd if keys is None: keys = self.default_keys - return pd.DataFrame.from_records( - {key: formatter(data, key) for key in keys} - for data in self.to_dicts() - ) + return [{key: formatter(data, key) for key in keys} for data in self.to_dicts()] def to_string( self, diff --git a/tests/test_utils.py b/tests/test_utils.py index 5c601e2..2032b69 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -183,14 +183,14 @@ def test_pretty_dicts_mixin_to_pandas(pretty_dicts: tea_tasting.utils.PrettyDict }), ) -def test_pretty_dicts_mixin_to_pretty(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - pd.testing.assert_frame_equal( - pretty_dicts.to_pretty(), - pd.DataFrame({ - "a": ("0.123", "0.346", "0.568"), - "b": ("0.235", "0.457", "0.679"), - }), - ) +def test_pretty_dicts_mixin_to_pretty_dicts( + pretty_dicts: tea_tasting.utils.PrettyDictsMixin, +): + assert pretty_dicts.to_pretty_dicts() == [ + {"a": "0.123", "b": "0.235"}, + {"a": "0.346", "b": "0.457"}, + {"a": "0.568", "b": "0.679"}, + ] def test_pretty_dicts_mixin_to_string(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): assert pretty_dicts.to_string() == textwrap.dedent("""\ From c30ea246a0566af6f0afa5bf2c77d743a6c89634 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 22:35:04 +0300 Subject: [PATCH 14/29] Rename PrettyDictsMixin -> DictsReprMixin --- src/tea_tasting/experiment.py | 6 +++--- src/tea_tasting/metrics/base.py | 2 +- src/tea_tasting/multiplicity.py | 2 +- src/tea_tasting/utils.py | 4 ++-- tests/test_experiment.py | 2 +- tests/test_utils.py | 38 ++++++++++++++++----------------- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/tea_tasting/experiment.py b/src/tea_tasting/experiment.py index 7578cd5..f00f606 100644 --- a/src/tea_tasting/experiment.py +++ b/src/tea_tasting/experiment.py @@ -22,7 +22,7 @@ class ExperimentResult( UserDict[str, tea_tasting.metrics.MetricResult], - tea_tasting.utils.PrettyDictsMixin, + tea_tasting.utils.DictsReprMixin, ): """Experiment result for a pair of variants.""" default_keys = ( @@ -84,7 +84,7 @@ def to_dicts(self) -> tuple[dict[str, Any], ...]: class ExperimentResults( UserDict[tuple[Any, Any], ExperimentResult], - tea_tasting.utils.PrettyDictsMixin, + tea_tasting.utils.DictsReprMixin, ): """Experiment results for multiple pairs of variants.""" default_keys = ( @@ -108,7 +108,7 @@ def to_dicts(self) -> tuple[dict[str, Any], ...]: class ExperimentPowerResult( UserDict[str, tea_tasting.metrics.MetricPowerResults[Any]], - tea_tasting.utils.PrettyDictsMixin, + tea_tasting.utils.DictsReprMixin, ): """Result of the analysis of power in a experiment.""" default_keys = ("metric", "power", "effect_size", "rel_effect_size", "n_obs") diff --git a/src/tea_tasting/metrics/base.py b/src/tea_tasting/metrics/base.py index fa6b388..6573190 100644 --- a/src/tea_tasting/metrics/base.py +++ b/src/tea_tasting/metrics/base.py @@ -30,7 +30,7 @@ P = TypeVar("P", bound=MetricPowerResult) -class MetricPowerResults(UserList[P], tea_tasting.utils.PrettyDictsMixin): +class MetricPowerResults(UserList[P], tea_tasting.utils.DictsReprMixin): """Power analysis results.""" default_keys = ("power", "effect_size", "rel_effect_size", "n_obs") diff --git a/src/tea_tasting/multiplicity.py b/src/tea_tasting/multiplicity.py index 8dc0896..f691b16 100644 --- a/src/tea_tasting/multiplicity.py +++ b/src/tea_tasting/multiplicity.py @@ -21,7 +21,7 @@ class MultipleComparisonsResults( UserDict[Any, tea_tasting.experiment.ExperimentResult], - tea_tasting.utils.PrettyDictsMixin, + tea_tasting.utils.DictsReprMixin, ): """Multiple comparisons result.""" default_keys = ( diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 630c449..f5c3eea 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -208,8 +208,8 @@ def get_and_format_num(data: dict[str, Any], key: str) -> str: return format_num(val, sig=sig, pct=pct) -class PrettyDictsMixin(abc.ABC): - """Pretty representation of a sequence of dictionaries. +class DictsReprMixin(abc.ABC): + """Representation and conversion of a sequence of dictionaries. Default formatting rules: - If a name starts with `"rel_"` or equals to `"power"` consider it diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 64266cf..24cab77 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -336,7 +336,7 @@ def test_experiment_power_result_to_dicts(): _PowerResult(**raw_results[3]), ]), }) - assert isinstance(result, tea_tasting.utils.PrettyDictsMixin) + assert isinstance(result, tea_tasting.utils.DictsReprMixin) assert result.default_keys == ( "metric", "power", "effect_size", "rel_effect_size", "n_obs") assert result.to_dicts() == ( diff --git a/tests/test_utils.py b/tests/test_utils.py index 2032b69..f3bf30a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -163,8 +163,8 @@ def test_get_and_format_num(): @pytest.fixture -def pretty_dicts() -> tea_tasting.utils.PrettyDictsMixin: - class PrettyDicts(tea_tasting.utils.PrettyDictsMixin): +def dicts_repr() -> tea_tasting.utils.DictsReprMixin: + class DictsRepr(tea_tasting.utils.DictsReprMixin): default_keys = ("a", "b") def to_dicts(self) -> tuple[dict[str, Any], ...]: return ( @@ -172,35 +172,35 @@ def to_dicts(self) -> tuple[dict[str, Any], ...]: {"a": 0.34567, "b": 0.45678}, {"a": 0.56789, "b": 0.67890}, ) - return PrettyDicts() + return DictsRepr() -def test_pretty_dicts_mixin_to_pandas(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): +def test_dicts_repr_mixin_to_pandas(dicts_repr: tea_tasting.utils.DictsReprMixin): pd.testing.assert_frame_equal( - pretty_dicts.to_pandas(), + dicts_repr.to_pandas(), pd.DataFrame({ "a": (0.12345, 0.34567, 0.56789), "b": (0.23456, 0.45678, 0.67890), }), ) -def test_pretty_dicts_mixin_to_pretty_dicts( - pretty_dicts: tea_tasting.utils.PrettyDictsMixin, +def test_dicts_repr_mixin_to_pretty_dicts( + dicts_repr: tea_tasting.utils.DictsReprMixin, ): - assert pretty_dicts.to_pretty_dicts() == [ + assert dicts_repr.to_pretty_dicts() == [ {"a": "0.123", "b": "0.235"}, {"a": "0.346", "b": "0.457"}, {"a": "0.568", "b": "0.679"}, ] -def test_pretty_dicts_mixin_to_string(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert pretty_dicts.to_string() == textwrap.dedent("""\ +def test_dicts_repr_mixin_to_string(dicts_repr: tea_tasting.utils.DictsReprMixin): + assert dicts_repr.to_string() == textwrap.dedent("""\ a b 0.123 0.235 0.346 0.457 0.568 0.679""") -def test_pretty_dicts_mixin_to_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert pretty_dicts.to_html() == ( +def test_dicts_repr_mixin_to_html(dicts_repr: tea_tasting.utils.DictsReprMixin): + assert dicts_repr.to_html() == ( '' '' '' @@ -210,10 +210,10 @@ def test_pretty_dicts_mixin_to_html(pretty_dicts: tea_tasting.utils.PrettyDictsM '
ab
' ) -def test_pretty_dicts_mixin_to_html_indent( - pretty_dicts: tea_tasting.utils.PrettyDictsMixin, +def test_dicts_repr_mixin_to_html_indent( + dicts_repr: tea_tasting.utils.DictsReprMixin, ): - assert pretty_dicts.to_html(indent=" ") == textwrap.dedent("""\ + assert dicts_repr.to_html(indent=" ") == textwrap.dedent("""\ @@ -237,15 +237,15 @@ def test_pretty_dicts_mixin_to_html_indent(
""") -def test_pretty_dicts_mixin_str(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert str(pretty_dicts) == textwrap.dedent("""\ +def test_dicts_repr_mixin_str(dicts_repr: tea_tasting.utils.DictsReprMixin): + assert str(dicts_repr) == textwrap.dedent("""\ a b 0.123 0.235 0.346 0.457 0.568 0.679""") -def test_pretty_dicts_mixin_repr_html(pretty_dicts: tea_tasting.utils.PrettyDictsMixin): - assert pretty_dicts._repr_html_() == ( +def test_dicts_repr_mixin_repr_html(dicts_repr: tea_tasting.utils.DictsReprMixin): + assert dicts_repr._repr_html_() == ( '' '' '' From 438545fbd6e5b45348feef050b50c1ef99210815 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 22:37:11 +0300 Subject: [PATCH 15/29] Minor fix --- src/tea_tasting/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index f5c3eea..9ac59de 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -236,7 +236,7 @@ def to_pretty_dicts( self, keys: Sequence[str] | None = None, formatter: Callable[[dict[str, Any], str], str] = get_and_format_num, - ) -> list[dict[str, Any]]: + ) -> list[dict[str, str]]: """Convert the object to a list of dictionaries with formatted values. Args: From 5b89b69e63e2ffb2751b1dac307054cffb21c336 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 23:05:44 +0300 Subject: [PATCH 16/29] Change type of the variant column in the sample dataset --- src/tea_tasting/datasets.py | 2 +- tests/metrics/test_base.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/tea_tasting/datasets.py b/src/tea_tasting/datasets.py index 62d5853..2678312 100644 --- a/src/tea_tasting/datasets.py +++ b/src/tea_tasting/datasets.py @@ -383,7 +383,7 @@ def _make_data( data = { "user": user, - "variant": variant[user].astype(np.uint8), + "variant": variant[user], "sessions": sessions, "orders": orders, "revenue": revenue, diff --git a/tests/metrics/test_base.py b/tests/metrics/test_base.py index 940d224..dda2628 100644 --- a/tests/metrics/test_base.py +++ b/tests/metrics/test_base.py @@ -6,7 +6,6 @@ import ibis import pandas as pd import polars as pl -import pyarrow as pa import pytest import tea_tasting.aggr @@ -18,6 +17,7 @@ from typing import Literal import ibis.expr.types # noqa: TC004 + import pyarrow as pa Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame @@ -67,12 +67,7 @@ def test_aggr_cols_len(): @pytest.fixture def data_arrow() -> pa.Table: - table = tea_tasting.datasets.make_users_data(n_users=100, seed=42) - return table.set_column( - table.schema.get_field_index("variant"), - "variant", - table["variant"].cast(pa.int64()), - ) + return tea_tasting.datasets.make_users_data(n_users=100, seed=42) @pytest.fixture def data_pandas(data_arrow: pa.Table) -> pd.DataFrame: From 2b8031c5fbf54d0c1dd35da9951f4d4720a67190 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Fri, 27 Dec 2024 23:09:34 +0300 Subject: [PATCH 17/29] Remove pandas dependency in read_aggregates --- src/tea_tasting/aggr.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/tea_tasting/aggr.py b/src/tea_tasting/aggr.py index 5121d4a..6588a97 100644 --- a/src/tea_tasting/aggr.py +++ b/src/tea_tasting/aggr.py @@ -17,7 +17,6 @@ from typing import Any import narwhals.typing # noqa: TC004 - import pandas as pd _COUNT = "_count" @@ -302,7 +301,7 @@ def read_aggregates( if group_col is None: return _get_aggregates( - aggr_data, + aggr_data[0], has_count=has_count, mean_cols=mean_cols, var_cols=var_cols, @@ -310,14 +309,14 @@ def read_aggregates( ) return { - group: _get_aggregates( + group_data[group_col]: _get_aggregates( group_data, has_count=has_count, mean_cols=mean_cols, var_cols=var_cols, cov_cols=cov_cols, ) - for group, group_data in aggr_data.groupby(group_col) + for group_data in aggr_data } @@ -349,7 +348,7 @@ def _read_aggr_ibis( mean_cols: Sequence[str], var_cols: Sequence[str], cov_cols: Sequence[tuple[str, str]], -) -> pd.DataFrame: +) -> list[dict[str, Any]]: covar_cols = tuple({*var_cols, *itertools.chain(*cov_cols)}) backend = ibis.get_backend(data) var_op = ibis.expr.operations.Variance @@ -394,7 +393,7 @@ def _read_aggr_ibis( all_expr = count_expr | mean_expr | var_expr | cov_expr grouped_data = data.group_by(group_col) if group_col is not None else data # type: ignore - return grouped_data.aggregate(**all_expr).to_pandas() # type: ignore + return grouped_data.aggregate(**all_expr).to_pyarrow().to_pylist() # type: ignore def _read_aggr_narwhals( @@ -405,7 +404,7 @@ def _read_aggr_narwhals( mean_cols: Sequence[str], var_cols: Sequence[str], cov_cols: Sequence[tuple[str, str]], -) -> pd.DataFrame: +) -> list[dict[str, Any]]: data = nw.from_native(data) if not isinstance(data, nw.LazyFrame): data = data.lazy() @@ -457,7 +456,7 @@ def _read_aggr_narwhals( }, ) - return aggr_data.collect().to_pandas() + return aggr_data.collect().to_arrow().to_pylist() def _demean_nw_col(col: str, group_col: str | None) -> nw.Expr: @@ -467,17 +466,16 @@ def _demean_nw_col(col: str, group_col: str | None) -> nw.Expr: def _get_aggregates( - data: pd.DataFrame, + data: dict[str, Any], *, has_count: bool, mean_cols: Sequence[str], var_cols: Sequence[str], cov_cols: Sequence[tuple[str, str]], ) -> Aggregates: - s = data.iloc[0] return Aggregates( - count_=s[_COUNT] if has_count else None, - mean_={col: s[_MEAN.format(col)] for col in mean_cols}, - var_={col: s[_VAR.format(col)] for col in var_cols}, - cov_={cols: s[_COV.format(*cols)] for cols in cov_cols}, + count_=data[_COUNT] if has_count else None, + mean_={col: data[_MEAN.format(col)] for col in mean_cols}, + var_={col: data[_VAR.format(col)] for col in var_cols}, + cov_={cols: data[_COV.format(*cols)] for cols in cov_cols}, ) From 6a78558794f58a1ae3ffecd6786bb521facaa635 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 00:15:26 +0300 Subject: [PATCH 18/29] Switch from pandas to pyarrow in base metrics and experiment --- src/tea_tasting/experiment.py | 19 +++---- src/tea_tasting/metrics/__init__.py | 2 +- src/tea_tasting/metrics/base.py | 46 ++++++++-------- src/tea_tasting/metrics/resampling.py | 24 ++++++--- tests/metrics/test_base.py | 76 +++++++++++++++------------ tests/metrics/test_resampling.py | 15 +++--- tests/test_experiment.py | 11 ++-- 7 files changed, 105 insertions(+), 88 deletions(-) diff --git a/src/tea_tasting/experiment.py b/src/tea_tasting/experiment.py index f00f606..07d9445 100644 --- a/src/tea_tasting/experiment.py +++ b/src/tea_tasting/experiment.py @@ -7,7 +7,6 @@ import ibis.expr.types import narwhals as nw -import pandas as pd import tea_tasting.aggr import tea_tasting.metrics @@ -18,6 +17,7 @@ from typing import Literal import narwhals.typing # noqa: TC004 + import pyarrow as pa class ExperimentResult( @@ -322,7 +322,7 @@ def _analyze_metric( metric: tea_tasting.metrics.MetricBase[Any], data: narwhals.typing.IntoFrame | ibis.expr.types.Table, aggregated_data: dict[Any, tea_tasting.aggr.Aggregates] | None, - granular_data: dict[Any, pd.DataFrame] | None, + granular_data: dict[Any, pa.Table] | None, control: Any, treatment: Any, ) -> tea_tasting.metrics.MetricResult: @@ -346,7 +346,7 @@ def _read_data( data: narwhals.typing.IntoFrame | ibis.expr.types.Table, ) -> tuple[ dict[Any, tea_tasting.aggr.Aggregates] | None, - dict[Any, pd.DataFrame] | None, + dict[Any, pa.Table] | None, ]: aggr_cols = tea_tasting.metrics.AggrCols() gran_cols = set() @@ -362,7 +362,7 @@ def _read_data( variant=self.variant, ) if len(aggr_cols) > 0 else None - granular_data = tea_tasting.metrics.read_dataframes( + granular_data = tea_tasting.metrics.read_granular( data, cols=tuple(gran_cols), variant=self.variant, @@ -374,22 +374,19 @@ def _read_data( def _read_variants( self, data: narwhals.typing.IntoFrame | ibis.expr.types.Table, - ) -> pd.Series[Any]: # type: ignore - if isinstance(data, pd.DataFrame): - return data.loc[:, self.variant].drop_duplicates() # type: ignore - + ) -> list[Any]: if isinstance(data, ibis.expr.types.Table): return ( data.select(self.variant) .distinct() - .to_pandas() - .loc[:, self.variant] + .to_pyarrow()[self.variant] + .to_pylist() ) data = nw.from_native(data) if not isinstance(data, nw.LazyFrame): data = data.lazy() - return data.unique(self.variant).collect().to_pandas().loc[:, self.variant] + return data.unique(self.variant).collect().get_column(self.variant).to_list() def solve_power( diff --git a/src/tea_tasting/metrics/__init__.py b/src/tea_tasting/metrics/__init__.py index c096f3c..1e3a76f 100644 --- a/src/tea_tasting/metrics/__init__.py +++ b/src/tea_tasting/metrics/__init__.py @@ -20,7 +20,7 @@ PowerBase, PowerBaseAggregated, aggregate_by_variants, - read_dataframes, + read_granular, ) from tea_tasting.metrics.mean import Mean, RatioOfMeans from tea_tasting.metrics.proportion import SampleRatio diff --git a/src/tea_tasting/metrics/base.py b/src/tea_tasting/metrics/base.py index 6573190..effd997 100644 --- a/src/tea_tasting/metrics/base.py +++ b/src/tea_tasting/metrics/base.py @@ -9,7 +9,8 @@ import ibis import ibis.expr.types import narwhals as nw -import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc import tea_tasting.aggr import tea_tasting.utils @@ -303,7 +304,7 @@ class MetricBaseGranular(MetricBase[R], _HasCols): @overload def analyze( self, - data: dict[Any, pd.DataFrame], + data: dict[Any, pa.Table], control: Any, treatment: Any, variant: str | None = None, @@ -325,7 +326,7 @@ def analyze( data: ( narwhals.typing.IntoFrame | ibis.expr.types.Table | - dict[Any, pd.DataFrame] + dict[Any, pa.Table] ), control: Any, treatment: Any, @@ -342,21 +343,21 @@ def analyze( Returns: Analysis result. """ - dfs = read_dataframes( + dfs = read_granular( data, cols=self.cols, variant=variant, ) - return self.analyze_dataframes( + return self.analyze_granular( control=dfs[control], treatment=dfs[treatment], ) @abc.abstractmethod - def analyze_dataframes( + def analyze_granular( self, - control: pd.DataFrame, - treatment: pd.DataFrame, + control: pa.Table, + treatment: pa.Table, ) -> R: """Analyze metric in an experiment using granular data. @@ -369,11 +370,11 @@ def analyze_dataframes( """ -def read_dataframes( - data: narwhals.typing.IntoFrame | ibis.expr.types.Table | dict[Any, pd.DataFrame], +def read_granular( + data: narwhals.typing.IntoFrame | ibis.expr.types.Table | dict[Any, pa.Table], cols: Sequence[str], variant: str | None = None, -) -> dict[Any, pd.DataFrame]: +) -> dict[Any, pa.Table]: """Read granular experimental data. Args: @@ -383,28 +384,29 @@ def read_dataframes( Raises: ValueError: The variant parameter is required but was not provided. - TypeError: data is not an instance of DataFrame, Table, - or a dictionary if DataFrames. Returns: - Experimental data as a dictionary of DataFrames. + Experimental data as a dictionary of PyArrow Tables. """ if isinstance(data, dict) and all( - isinstance(v, pd.DataFrame) for v in data.values() # type: ignore + isinstance(v, pa.Table) for v in data.values() ): return data if variant is None: raise ValueError("The variant parameter is required but was not provided.") - if isinstance(data, pd.DataFrame): - data = data.loc[:, [*cols, variant]] - elif isinstance(data, ibis.expr.types.Table): - data = data.select(*cols, variant).to_pandas() + if isinstance(data, ibis.expr.types.Table): + table = data.select(*cols, variant).to_pyarrow() else: data = nw.from_native(data) if not isinstance(data, nw.LazyFrame): data = data.lazy() - data = data.select(*cols, variant).collect().to_pandas() - - return dict(tuple(data.groupby(variant))) # type: ignore + table = data.select(*cols, variant).collect().to_arrow() + + variant_col = table[variant] + table = table.select(cols) + return { + var: table.filter(pc.equal(variant_col, pa.scalar(var))) # type: ignore + for var in variant_col.unique().to_pylist() + } diff --git a/src/tea_tasting/metrics/resampling.py b/src/tea_tasting/metrics/resampling.py index abb8ac7..e45ba76 100644 --- a/src/tea_tasting/metrics/resampling.py +++ b/src/tea_tasting/metrics/resampling.py @@ -18,7 +18,7 @@ from typing import Any, Literal import numpy.typing as npt - import pandas as pd + import pyarrow as pa class BootstrapResult(NamedTuple): @@ -195,10 +195,10 @@ def cols(self) -> Sequence[str]: return self.columns - def analyze_dataframes( + def analyze_granular( self, - control: pd.DataFrame, - treatment: pd.DataFrame, + control: pa.Table, + treatment: pa.Table, ) -> BootstrapResult: """Analyze metric in an experiment using granular data. @@ -223,8 +223,8 @@ def statistic( return np.stack((effect_size, rel_effect_size), axis=0) - contr = control.loc[:, self.columns].to_numpy() # type: ignore - treat = treatment.loc[:, self.columns].to_numpy() # type: ignore + contr = _select_as_numpy(control, self.columns) + treat = _select_as_numpy(treatment, self.columns) stat = statistic(contr, treat, axis=0) result = scipy.stats.bootstrap( @@ -252,6 +252,18 @@ def statistic( ) +def _select_as_numpy( + data: pa.Table, + columns: str | Sequence[str], +) -> npt.NDArray[np.number[Any]]: + if isinstance(columns, str): + columns = (columns,) + return np.column_stack([ + data[col].combine_chunks().to_numpy(zero_copy_only=False) + for col in columns + ]) + + class Quantile(Bootstrap): # noqa: D101 def __init__( self, diff --git a/tests/metrics/test_base.py b/tests/metrics/test_base.py index dda2628..881b46e 100644 --- a/tests/metrics/test_base.py +++ b/tests/metrics/test_base.py @@ -4,8 +4,9 @@ import unittest.mock import ibis -import pandas as pd import polars as pl +import pyarrow as pa +import pyarrow.compute as pc import pytest import tea_tasting.aggr @@ -17,7 +18,7 @@ from typing import Literal import ibis.expr.types # noqa: TC004 - import pyarrow as pa + import pandas as pd Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame @@ -134,11 +135,16 @@ def cols() -> tuple[str, ...]: return ("sessions", "orders", "revenue") @pytest.fixture -def correct_dfs( - data_pandas: pd.DataFrame, +def correct_gran( + data_arrow: pa.Table, cols: tuple[str, ...], -) -> dict[Any, pd.DataFrame]: - return dict(tuple(data_pandas.loc[:, [*cols, "variant"]].groupby("variant"))) +) -> dict[Any, pa.Table]: + variant_col = data_arrow["variant"] + table = data_arrow.select(cols) + return { + var: table.filter(pc.equal(variant_col, pa.scalar(var))) # type: ignore + for var in variant_col.unique().to_pylist() + } @pytest.fixture def aggr_metric( @@ -195,7 +201,7 @@ class GranMetric(tea_tasting.metrics.base.MetricBaseGranular[dict[str, Any]]): def cols(self) -> tuple[str, ...]: return cols - def analyze_dataframes( + def analyze_granular( self, control: pd.DataFrame, # noqa: ARG002 treatment: pd.DataFrame, # noqa: ARG002 @@ -329,55 +335,55 @@ def test_aggregate_by_variants_raises( def test_metric_base_granular_frame( gran_metric: tea_tasting.metrics.base.MetricBaseGranular[dict[str, Any]], data_pandas: pd.DataFrame, - correct_dfs: dict[Any, pd.DataFrame], + correct_gran: dict[Any, pa.Table], ): - gran_metric.analyze_dataframes = unittest.mock.MagicMock() + gran_metric.analyze_granular = unittest.mock.MagicMock() gran_metric.analyze(data_pandas, control=0, treatment=1, variant="variant") - gran_metric.analyze_dataframes.assert_called_once() - kwargs = gran_metric.analyze_dataframes.call_args.kwargs - pd.testing.assert_frame_equal(kwargs["control"], correct_dfs[0]) - pd.testing.assert_frame_equal(kwargs["treatment"], correct_dfs[1]) + gran_metric.analyze_granular.assert_called_once() + kwargs = gran_metric.analyze_granular.call_args.kwargs + assert kwargs["control"].equals(correct_gran[0]) + assert kwargs["treatment"].equals(correct_gran[1]) -def test_metric_base_granular_dfs( +def test_metric_base_granular_gran( gran_metric: tea_tasting.metrics.base.MetricBaseGranular[dict[str, Any]], - correct_dfs: dict[Any, pd.DataFrame], + correct_gran: dict[Any, pd.DataFrame], ): - gran_metric.analyze_dataframes = unittest.mock.MagicMock() - gran_metric.analyze(correct_dfs, control=0, treatment=1) - gran_metric.analyze_dataframes.assert_called_once() - kwargs = gran_metric.analyze_dataframes.call_args.kwargs - pd.testing.assert_frame_equal(kwargs["control"], correct_dfs[0]) - pd.testing.assert_frame_equal(kwargs["treatment"], correct_dfs[1]) + gran_metric.analyze_granular = unittest.mock.MagicMock() + gran_metric.analyze(correct_gran, control=0, treatment=1) + gran_metric.analyze_granular.assert_called_once() + kwargs = gran_metric.analyze_granular.call_args.kwargs + assert kwargs["control"].equals(correct_gran[0]) + assert kwargs["treatment"].equals(correct_gran[1]) -def test_read_dataframes_frame( +def test_read_granular_frame( data: Frame, cols: tuple[str, ...], - correct_dfs: dict[Any, pd.DataFrame], + correct_gran: dict[Any, pa.Table], ): - dfs = tea_tasting.metrics.base.read_dataframes( + gran = tea_tasting.metrics.base.read_granular( data, cols=cols, variant="variant", ) - pd.testing.assert_frame_equal(dfs[0], correct_dfs[0]) - pd.testing.assert_frame_equal(dfs[1], correct_dfs[1]) + assert gran[0].equals(correct_gran[0]) + assert gran[1].equals(correct_gran[1]) -def test_read_dataframes_dfs( +def test_read_granular_dict( cols: tuple[str, ...], - correct_dfs: dict[Any, pd.DataFrame], + correct_gran: dict[Any, pa.Table], ): - dfs = tea_tasting.metrics.base.read_dataframes( - correct_dfs, + gran = tea_tasting.metrics.base.read_granular( + correct_gran, cols=cols, variant="variant", ) - pd.testing.assert_frame_equal(dfs[0], correct_dfs[0]) - pd.testing.assert_frame_equal(dfs[1], correct_dfs[1]) + assert gran[0].equals(correct_gran[0]) + assert gran[1].equals(correct_gran[1]) -def test_read_dataframes_raises( - data_pandas: ibis.expr.types.Table, +def test_read_granular_raises( + data_pandas: pd.DataFrame, cols: tuple[str, ...], ): with pytest.raises(ValueError, match="variant"): - tea_tasting.metrics.base.read_dataframes(data_pandas, cols=cols) + tea_tasting.metrics.base.read_granular(data_pandas, cols=cols) diff --git a/tests/metrics/test_resampling.py b/tests/metrics/test_resampling.py index 02ad643..01ed997 100644 --- a/tests/metrics/test_resampling.py +++ b/tests/metrics/test_resampling.py @@ -16,17 +16,16 @@ from typing import Any import numpy.typing as npt - import pandas as pd import pyarrow as pa @pytest.fixture -def data_arrow() -> pd.DataFrame: +def data_arrow() -> pa.Table: return tea_tasting.datasets.make_users_data(n_users=100, seed=42) @pytest.fixture -def data_gran(data_arrow: pa.Table) -> dict[Any, pd.DataFrame]: - return tea_tasting.metrics.base.read_dataframes( +def data_gran(data_arrow: pa.Table) -> dict[Any, pa.Table]: + return tea_tasting.metrics.base.read_granular( data_arrow, ("sessions", "orders", "revenue"), variant="variant", @@ -79,7 +78,7 @@ def test_bootstrap_analyze_frame(data_arrow: pa.Table): assert isinstance(result, tea_tasting.metrics.resampling.BootstrapResult) -def test_bootstrap_analyze_default(data_gran: dict[Any, pd.DataFrame]): +def test_bootstrap_analyze_default(data_gran: dict[Any, pa.Table]): metric = tea_tasting.metrics.resampling.Bootstrap( "revenue", np.mean, @@ -97,7 +96,7 @@ def test_bootstrap_analyze_default(data_gran: dict[Any, pd.DataFrame]): assert result.rel_effect_size_ci_lower == pytest.approx(-0.5658060166766641) assert result.rel_effect_size_ci_upper == pytest.approx(1.8185107973505807) -def test_bootstrap_analyze_multiple_columns(data_gran: dict[Any, pd.DataFrame]): +def test_bootstrap_analyze_multiple_columns(data_gran: dict[Any, pa.Table]): def ratio_of_means( sample: npt.NDArray[np.number[Any]], axis: int, @@ -122,7 +121,7 @@ def ratio_of_means( assert result.rel_effect_size_ci_lower == pytest.approx(-0.6424902672606227) assert result.rel_effect_size_ci_upper == pytest.approx(0.4374404130492657) -def test_bootstrap_analyze_division_by_zero(data_gran: dict[Any, pd.DataFrame]): +def test_bootstrap_analyze_division_by_zero(data_gran: dict[Any, pa.Table]): metric = tea_tasting.metrics.resampling.Bootstrap( "orders", np.median, @@ -141,7 +140,7 @@ def test_bootstrap_analyze_division_by_zero(data_gran: dict[Any, pd.DataFrame]): assert np.isnan(result.rel_effect_size_ci_lower) assert np.isnan(result.rel_effect_size_ci_upper) -def test_quantile(data_gran: dict[Any, pd.DataFrame]): +def test_quantile(data_gran: dict[Any, pa.Table]): metric = tea_tasting.metrics.resampling.Quantile( "revenue", q=0.8, diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 24cab77..51b9056 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -7,6 +7,7 @@ import narwhals as nw import pandas as pd import polars as pl +import pyarrow.compute as pc import pytest import tea_tasting.aggr @@ -132,13 +133,13 @@ def __init__(self, value: str) -> None: def cols(self) -> tuple[str, ...]: return (self.value,) - def analyze_dataframes( + def analyze_granular( self, - control: pd.DataFrame, - treatment: pd.DataFrame, + control: pa.Table, + treatment: pa.Table, ) -> _MetricResultDict: - contr_mean = control.loc[:, self.value].mean() - treat_mean = treatment.loc[:, self.value].mean() + contr_mean = pc.mean(control[self.value]).as_py() # type: ignore + treat_mean = pc.mean(treatment[self.value]).as_py() # type: ignore return _MetricResultDict( control=contr_mean, treatment=treat_mean, From de2b4becae82d403997017e2f8753caf4d04d900 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 00:31:28 +0300 Subject: [PATCH 19/29] Change and update dependencies --- pdm.lock | 222 ++++++++++++++++++++++++------------------------- pyproject.toml | 4 +- 2 files changed, 113 insertions(+), 113 deletions(-) diff --git a/pdm.lock b/pdm.lock index da08133..b6bd1c4 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "docs", "lint", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:fa57208e1cef2df5fe5f46f4ee2b285b71d8f41c1767d0ba8d31f7e010302a51" +content_hash = "sha256:8e11adabfb8f2725d292c52f63b33796d86fddb19d91c05f7a4e5e1468d839f1" [[metadata.targets]] requires_python = ">=3.10" @@ -130,129 +130,129 @@ files = [ [[package]] name = "coverage" -version = "7.6.9" +version = "7.6.10" requires_python = ">=3.9" summary = "Code coverage measurement for Python" groups = ["test"] files = [ - {file = "coverage-7.6.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb"}, - {file = "coverage-7.6.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96d636c77af18b5cb664ddf12dab9b15a0cfe9c0bde715da38698c8cea748bfa"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75cded8a3cff93da9edc31446872d2997e327921d8eed86641efafd350e1df1"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7b15f589593110ae767ce997775d645b47e5cbbf54fd322f8ebea6277466cec"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:44349150f6811b44b25574839b39ae35291f6496eb795b7366fef3bd3cf112d3"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d891c136b5b310d0e702e186d70cd16d1119ea8927347045124cb286b29297e5"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:db1dab894cc139f67822a92910466531de5ea6034ddfd2b11c0d4c6257168073"}, - {file = "coverage-7.6.9-cp310-cp310-win32.whl", hash = "sha256:41ff7b0da5af71a51b53f501a3bac65fb0ec311ebed1632e58fc6107f03b9198"}, - {file = "coverage-7.6.9-cp310-cp310-win_amd64.whl", hash = "sha256:35371f8438028fdccfaf3570b31d98e8d9eda8bb1d6ab9473f5a390969e98717"}, - {file = "coverage-7.6.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:932fc826442132dde42ee52cf66d941f581c685a6313feebed358411238f60f9"}, - {file = "coverage-7.6.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:085161be5f3b30fd9b3e7b9a8c301f935c8313dcf928a07b116324abea2c1c2c"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccc660a77e1c2bf24ddbce969af9447a9474790160cfb23de6be4fa88e3951c7"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c69e42c892c018cd3c8d90da61d845f50a8243062b19d228189b0224150018a9"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0824a28ec542a0be22f60c6ac36d679e0e262e5353203bea81d44ee81fe9c6d4"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4401ae5fc52ad8d26d2a5d8a7428b0f0c72431683f8e63e42e70606374c311a1"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:98caba4476a6c8d59ec1eb00c7dd862ba9beca34085642d46ed503cc2d440d4b"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ee5defd1733fd6ec08b168bd4f5387d5b322f45ca9e0e6c817ea6c4cd36313e3"}, - {file = "coverage-7.6.9-cp311-cp311-win32.whl", hash = "sha256:f2d1ec60d6d256bdf298cb86b78dd715980828f50c46701abc3b0a2b3f8a0dc0"}, - {file = "coverage-7.6.9-cp311-cp311-win_amd64.whl", hash = "sha256:0d59fd927b1f04de57a2ba0137166d31c1a6dd9e764ad4af552912d70428c92b"}, - {file = "coverage-7.6.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:99e266ae0b5d15f1ca8d278a668df6f51cc4b854513daab5cae695ed7b721cf8"}, - {file = "coverage-7.6.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9901d36492009a0a9b94b20e52ebfc8453bf49bb2b27bca2c9706f8b4f5a554a"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abd3e72dd5b97e3af4246cdada7738ef0e608168de952b837b8dd7e90341f015"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65dad5a248823a4996724a88eb51d4b31587aa7aa428562dbe459c684e5787ae"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:22be16571504c9ccea919fcedb459d5ab20d41172056206eb2994e2ff06118a4"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f957943bc718b87144ecaee70762bc2bc3f1a7a53c7b861103546d3a403f0a6"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ae1387db4aecb1f485fb70a6c0148c6cdaebb6038f1d40089b1fc84a5db556f"}, - {file = "coverage-7.6.9-cp312-cp312-win32.whl", hash = "sha256:1a330812d9cc7ac2182586f6d41b4d0fadf9be9049f350e0efb275c8ee8eb692"}, - {file = "coverage-7.6.9-cp312-cp312-win_amd64.whl", hash = "sha256:b12c6b18269ca471eedd41c1b6a1065b2f7827508edb9a7ed5555e9a56dcfc97"}, - {file = "coverage-7.6.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:899b8cd4781c400454f2f64f7776a5d87bbd7b3e7f7bda0cb18f857bb1334664"}, - {file = "coverage-7.6.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:61f70dc68bd36810972e55bbbe83674ea073dd1dcc121040a08cdf3416c5349c"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a289d23d4c46f1a82d5db4abeb40b9b5be91731ee19a379d15790e53031c014"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e216d8044a356fc0337c7a2a0536d6de07888d7bcda76febcb8adc50bdbbd00"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c026eb44f744acaa2bda7493dad903aa5bf5fc4f2554293a798d5606710055d"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e77363e8425325384f9d49272c54045bbed2f478e9dd698dbc65dbc37860eb0a"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:777abfab476cf83b5177b84d7486497e034eb9eaea0d746ce0c1268c71652077"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:447af20e25fdbe16f26e84eb714ba21d98868705cb138252d28bc400381f6ffb"}, - {file = "coverage-7.6.9-cp313-cp313-win32.whl", hash = "sha256:d872ec5aeb086cbea771c573600d47944eea2dcba8be5f3ee649bfe3cb8dc9ba"}, - {file = "coverage-7.6.9-cp313-cp313-win_amd64.whl", hash = "sha256:fd1213c86e48dfdc5a0cc676551db467495a95a662d2396ecd58e719191446e1"}, - {file = "coverage-7.6.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9e7484d286cd5a43744e5f47b0b3fb457865baf07bafc6bee91896364e1419"}, - {file = "coverage-7.6.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1cf0872ee455c03e5674b5bca5e3e68e159379c1af0903e89f5eba9ccc3a"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d10e07aa2b91835d6abec555ec8b2733347956991901eea6ffac295f83a30e4"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13a9e2d3ee855db3dd6ea1ba5203316a1b1fd8eaeffc37c5b54987e61e4194ae"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c38bf15a40ccf5619fa2fe8f26106c7e8e080d7760aeccb3722664c8656b030"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d5275455b3e4627c8e7154feaf7ee0743c2e7af82f6e3b561967b1cca755a0be"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8f8770dfc6e2c6a2d4569f411015c8d751c980d17a14b0530da2d7f27ffdd88e"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8d2dfa71665a29b153a9681edb1c8d9c1ea50dfc2375fb4dac99ea7e21a0bcd9"}, - {file = "coverage-7.6.9-cp313-cp313t-win32.whl", hash = "sha256:5e6b86b5847a016d0fbd31ffe1001b63355ed309651851295315031ea7eb5a9b"}, - {file = "coverage-7.6.9-cp313-cp313t-win_amd64.whl", hash = "sha256:97ddc94d46088304772d21b060041c97fc16bdda13c6c7f9d8fcd8d5ae0d8611"}, - {file = "coverage-7.6.9-pp39.pp310-none-any.whl", hash = "sha256:f3ca78518bc6bc92828cd11867b121891d75cae4ea9e908d72030609b996db1b"}, - {file = "coverage-7.6.9.tar.gz", hash = "sha256:4a8d8977b0c6ef5aeadcb644da9e69ae0dcfe66ec7f368c89c72e058bd71164d"}, + {file = "coverage-7.6.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78"}, + {file = "coverage-7.6.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5"}, + {file = "coverage-7.6.10-cp310-cp310-win32.whl", hash = "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244"}, + {file = "coverage-7.6.10-cp310-cp310-win_amd64.whl", hash = "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e"}, + {file = "coverage-7.6.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3"}, + {file = "coverage-7.6.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377"}, + {file = "coverage-7.6.10-cp311-cp311-win32.whl", hash = "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8"}, + {file = "coverage-7.6.10-cp311-cp311-win_amd64.whl", hash = "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609"}, + {file = "coverage-7.6.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853"}, + {file = "coverage-7.6.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852"}, + {file = "coverage-7.6.10-cp312-cp312-win32.whl", hash = "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359"}, + {file = "coverage-7.6.10-cp312-cp312-win_amd64.whl", hash = "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247"}, + {file = "coverage-7.6.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9"}, + {file = "coverage-7.6.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694"}, + {file = "coverage-7.6.10-cp313-cp313-win32.whl", hash = "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6"}, + {file = "coverage-7.6.10-cp313-cp313-win_amd64.whl", hash = "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e"}, + {file = "coverage-7.6.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe"}, + {file = "coverage-7.6.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2"}, + {file = "coverage-7.6.10-cp313-cp313t-win32.whl", hash = "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312"}, + {file = "coverage-7.6.10-cp313-cp313t-win_amd64.whl", hash = "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d"}, + {file = "coverage-7.6.10-pp39.pp310-none-any.whl", hash = "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f"}, + {file = "coverage-7.6.10.tar.gz", hash = "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23"}, ] [[package]] name = "coverage" -version = "7.6.9" +version = "7.6.10" extras = ["toml"] requires_python = ">=3.9" summary = "Code coverage measurement for Python" groups = ["test"] dependencies = [ - "coverage==7.6.9", + "coverage==7.6.10", "tomli; python_full_version <= \"3.11.0a6\"", ] files = [ - {file = "coverage-7.6.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb"}, - {file = "coverage-7.6.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96d636c77af18b5cb664ddf12dab9b15a0cfe9c0bde715da38698c8cea748bfa"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75cded8a3cff93da9edc31446872d2997e327921d8eed86641efafd350e1df1"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7b15f589593110ae767ce997775d645b47e5cbbf54fd322f8ebea6277466cec"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:44349150f6811b44b25574839b39ae35291f6496eb795b7366fef3bd3cf112d3"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d891c136b5b310d0e702e186d70cd16d1119ea8927347045124cb286b29297e5"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:db1dab894cc139f67822a92910466531de5ea6034ddfd2b11c0d4c6257168073"}, - {file = "coverage-7.6.9-cp310-cp310-win32.whl", hash = "sha256:41ff7b0da5af71a51b53f501a3bac65fb0ec311ebed1632e58fc6107f03b9198"}, - {file = "coverage-7.6.9-cp310-cp310-win_amd64.whl", hash = "sha256:35371f8438028fdccfaf3570b31d98e8d9eda8bb1d6ab9473f5a390969e98717"}, - {file = "coverage-7.6.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:932fc826442132dde42ee52cf66d941f581c685a6313feebed358411238f60f9"}, - {file = "coverage-7.6.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:085161be5f3b30fd9b3e7b9a8c301f935c8313dcf928a07b116324abea2c1c2c"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccc660a77e1c2bf24ddbce969af9447a9474790160cfb23de6be4fa88e3951c7"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c69e42c892c018cd3c8d90da61d845f50a8243062b19d228189b0224150018a9"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0824a28ec542a0be22f60c6ac36d679e0e262e5353203bea81d44ee81fe9c6d4"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4401ae5fc52ad8d26d2a5d8a7428b0f0c72431683f8e63e42e70606374c311a1"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:98caba4476a6c8d59ec1eb00c7dd862ba9beca34085642d46ed503cc2d440d4b"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ee5defd1733fd6ec08b168bd4f5387d5b322f45ca9e0e6c817ea6c4cd36313e3"}, - {file = "coverage-7.6.9-cp311-cp311-win32.whl", hash = "sha256:f2d1ec60d6d256bdf298cb86b78dd715980828f50c46701abc3b0a2b3f8a0dc0"}, - {file = "coverage-7.6.9-cp311-cp311-win_amd64.whl", hash = "sha256:0d59fd927b1f04de57a2ba0137166d31c1a6dd9e764ad4af552912d70428c92b"}, - {file = "coverage-7.6.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:99e266ae0b5d15f1ca8d278a668df6f51cc4b854513daab5cae695ed7b721cf8"}, - {file = "coverage-7.6.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9901d36492009a0a9b94b20e52ebfc8453bf49bb2b27bca2c9706f8b4f5a554a"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abd3e72dd5b97e3af4246cdada7738ef0e608168de952b837b8dd7e90341f015"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65dad5a248823a4996724a88eb51d4b31587aa7aa428562dbe459c684e5787ae"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:22be16571504c9ccea919fcedb459d5ab20d41172056206eb2994e2ff06118a4"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f957943bc718b87144ecaee70762bc2bc3f1a7a53c7b861103546d3a403f0a6"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ae1387db4aecb1f485fb70a6c0148c6cdaebb6038f1d40089b1fc84a5db556f"}, - {file = "coverage-7.6.9-cp312-cp312-win32.whl", hash = "sha256:1a330812d9cc7ac2182586f6d41b4d0fadf9be9049f350e0efb275c8ee8eb692"}, - {file = "coverage-7.6.9-cp312-cp312-win_amd64.whl", hash = "sha256:b12c6b18269ca471eedd41c1b6a1065b2f7827508edb9a7ed5555e9a56dcfc97"}, - {file = "coverage-7.6.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:899b8cd4781c400454f2f64f7776a5d87bbd7b3e7f7bda0cb18f857bb1334664"}, - {file = "coverage-7.6.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:61f70dc68bd36810972e55bbbe83674ea073dd1dcc121040a08cdf3416c5349c"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a289d23d4c46f1a82d5db4abeb40b9b5be91731ee19a379d15790e53031c014"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e216d8044a356fc0337c7a2a0536d6de07888d7bcda76febcb8adc50bdbbd00"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c026eb44f744acaa2bda7493dad903aa5bf5fc4f2554293a798d5606710055d"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e77363e8425325384f9d49272c54045bbed2f478e9dd698dbc65dbc37860eb0a"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:777abfab476cf83b5177b84d7486497e034eb9eaea0d746ce0c1268c71652077"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:447af20e25fdbe16f26e84eb714ba21d98868705cb138252d28bc400381f6ffb"}, - {file = "coverage-7.6.9-cp313-cp313-win32.whl", hash = "sha256:d872ec5aeb086cbea771c573600d47944eea2dcba8be5f3ee649bfe3cb8dc9ba"}, - {file = "coverage-7.6.9-cp313-cp313-win_amd64.whl", hash = "sha256:fd1213c86e48dfdc5a0cc676551db467495a95a662d2396ecd58e719191446e1"}, - {file = "coverage-7.6.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9e7484d286cd5a43744e5f47b0b3fb457865baf07bafc6bee91896364e1419"}, - {file = "coverage-7.6.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1cf0872ee455c03e5674b5bca5e3e68e159379c1af0903e89f5eba9ccc3a"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d10e07aa2b91835d6abec555ec8b2733347956991901eea6ffac295f83a30e4"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13a9e2d3ee855db3dd6ea1ba5203316a1b1fd8eaeffc37c5b54987e61e4194ae"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c38bf15a40ccf5619fa2fe8f26106c7e8e080d7760aeccb3722664c8656b030"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d5275455b3e4627c8e7154feaf7ee0743c2e7af82f6e3b561967b1cca755a0be"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8f8770dfc6e2c6a2d4569f411015c8d751c980d17a14b0530da2d7f27ffdd88e"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8d2dfa71665a29b153a9681edb1c8d9c1ea50dfc2375fb4dac99ea7e21a0bcd9"}, - {file = "coverage-7.6.9-cp313-cp313t-win32.whl", hash = "sha256:5e6b86b5847a016d0fbd31ffe1001b63355ed309651851295315031ea7eb5a9b"}, - {file = "coverage-7.6.9-cp313-cp313t-win_amd64.whl", hash = "sha256:97ddc94d46088304772d21b060041c97fc16bdda13c6c7f9d8fcd8d5ae0d8611"}, - {file = "coverage-7.6.9-pp39.pp310-none-any.whl", hash = "sha256:f3ca78518bc6bc92828cd11867b121891d75cae4ea9e908d72030609b996db1b"}, - {file = "coverage-7.6.9.tar.gz", hash = "sha256:4a8d8977b0c6ef5aeadcb644da9e69ae0dcfe66ec7f368c89c72e058bd71164d"}, + {file = "coverage-7.6.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78"}, + {file = "coverage-7.6.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5"}, + {file = "coverage-7.6.10-cp310-cp310-win32.whl", hash = "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244"}, + {file = "coverage-7.6.10-cp310-cp310-win_amd64.whl", hash = "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e"}, + {file = "coverage-7.6.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3"}, + {file = "coverage-7.6.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377"}, + {file = "coverage-7.6.10-cp311-cp311-win32.whl", hash = "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8"}, + {file = "coverage-7.6.10-cp311-cp311-win_amd64.whl", hash = "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609"}, + {file = "coverage-7.6.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853"}, + {file = "coverage-7.6.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852"}, + {file = "coverage-7.6.10-cp312-cp312-win32.whl", hash = "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359"}, + {file = "coverage-7.6.10-cp312-cp312-win_amd64.whl", hash = "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247"}, + {file = "coverage-7.6.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9"}, + {file = "coverage-7.6.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694"}, + {file = "coverage-7.6.10-cp313-cp313-win32.whl", hash = "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6"}, + {file = "coverage-7.6.10-cp313-cp313-win_amd64.whl", hash = "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e"}, + {file = "coverage-7.6.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe"}, + {file = "coverage-7.6.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2"}, + {file = "coverage-7.6.10-cp313-cp313t-win32.whl", hash = "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312"}, + {file = "coverage-7.6.10-cp313-cp313t-win_amd64.whl", hash = "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d"}, + {file = "coverage-7.6.10-pp39.pp310-none-any.whl", hash = "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f"}, + {file = "coverage-7.6.10.tar.gz", hash = "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23"}, ] [[package]] @@ -793,7 +793,7 @@ name = "pandas" version = "2.2.3" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" -groups = ["default", "test"] +groups = ["test"] dependencies = [ "numpy>=1.22.4; python_version < \"3.11\"", "numpy>=1.23.2; python_version == \"3.11\"", @@ -905,7 +905,7 @@ name = "pyarrow" version = "17.0.0" requires_python = ">=3.8" summary = "Python library for Apache Arrow" -groups = ["test"] +groups = ["default", "test"] dependencies = [ "numpy>=1.16.6", ] @@ -1311,7 +1311,7 @@ name = "tzdata" version = "2024.1" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default", "test"] +groups = ["test"] files = [ {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, diff --git a/pyproject.toml b/pyproject.toml index f930725..bb5baf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "ibis-framework>=7", "narwhals>=1", "numpy>=1.25", - "pandas>=2", + "pyarrow>=15", "scipy>=1.10", ] requires-python = ">=3.10" @@ -53,7 +53,7 @@ package-dir = "src" [tool.pdm.dev-dependencies] docs = ["mkdocs-material", "mkdocstrings[crystal,python]"] lint = ["pyright", "ruff"] -test = ["coverage[toml]", "ibis-framework[duckdb,sqlite]", "polars", "pytest"] +test = ["coverage[toml]", "ibis-framework[duckdb,sqlite]", "pandas", "polars", "pytest"] [tool.pdm.scripts] lint = "ruff check ." From bd3d4b07e0e6283b122c7336082bf4a4de78dac1 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 21:00:12 +0300 Subject: [PATCH 20/29] Add to_arrow and to_polars --- src/tea_tasting/utils.py | 22 +++++++++++++++++++--- tests/test_utils.py | 21 ++++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/tea_tasting/utils.py b/src/tea_tasting/utils.py index 9ac59de..5e8c800 100644 --- a/src/tea_tasting/utils.py +++ b/src/tea_tasting/utils.py @@ -11,15 +11,22 @@ from typing import TYPE_CHECKING import xml.etree.ElementTree as ET +import pyarrow as pa + if TYPE_CHECKING: from collections.abc import Callable, Iterator from typing import Any, Literal, TypeVar try: - from pandas import DataFrame + from pandas import DataFrame as PandasDataFrame + except ImportError: + from typing import Any as PandasDataFrame + + try: + from polars import DataFrame as PolarsDataFrame except ImportError: - from typing import Any as DataFrame + from typing import Any as PolarsDataFrame R = TypeVar("R") @@ -227,11 +234,20 @@ class DictsReprMixin(abc.ABC): def to_dicts(self) -> Sequence[dict[str, Any]]: """Convert the object to a sequence of dictionaries.""" - def to_pandas(self) -> DataFrame: + def to_arrow(self) -> pa.Table: + """Convert the object to a PyArrow Table.""" + return pa.Table.from_pylist(self.to_dicts()) + + def to_pandas(self) -> PandasDataFrame: """Convert the object to a Pandas DataFrame.""" import pandas as pd return pd.DataFrame.from_records(self.to_dicts()) + def to_polars(self) -> PolarsDataFrame: + """Convert the object to a Polars DataFrame.""" + import polars as pl + return pl.from_dicts(self.to_dicts()) + def to_pretty_dicts( self, keys: Sequence[str] | None = None, diff --git a/tests/test_utils.py b/tests/test_utils.py index f3bf30a..61d137d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,10 @@ from typing import TYPE_CHECKING import pandas as pd +import pandas.testing +import polars as pl +import polars.testing +import pyarrow as pa import pytest import tea_tasting.utils @@ -174,8 +178,14 @@ def to_dicts(self) -> tuple[dict[str, Any], ...]: ) return DictsRepr() +def test_dicts_repr_mixin_to_arrow(dicts_repr: tea_tasting.utils.DictsReprMixin): + assert dicts_repr.to_arrow().equals(pa.table({ + "a": (0.12345, 0.34567, 0.56789), + "b": (0.23456, 0.45678, 0.67890), + })) + def test_dicts_repr_mixin_to_pandas(dicts_repr: tea_tasting.utils.DictsReprMixin): - pd.testing.assert_frame_equal( + pandas.testing.assert_frame_equal( dicts_repr.to_pandas(), pd.DataFrame({ "a": (0.12345, 0.34567, 0.56789), @@ -183,6 +193,15 @@ def test_dicts_repr_mixin_to_pandas(dicts_repr: tea_tasting.utils.DictsReprMixin }), ) +def test_dicts_repr_mixin_to_polars(dicts_repr: tea_tasting.utils.DictsReprMixin): + polars.testing.assert_frame_equal( + dicts_repr.to_polars(), + pl.DataFrame({ + "a": (0.12345, 0.34567, 0.56789), + "b": (0.23456, 0.45678, 0.67890), + }), + ) + def test_dicts_repr_mixin_to_pretty_dicts( dicts_repr: tea_tasting.utils.DictsReprMixin, ): From d52d0e359d824f590a4951545d5332fb1d212209 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 21:19:56 +0300 Subject: [PATCH 21/29] Add `result_type` to `make_users_data` and `make_sessions_data` --- src/tea_tasting/datasets.py | 92 ++++++++++++++++++++++++++++--------- tests/test_datasets.py | 23 +++++++++- 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/src/tea_tasting/datasets.py b/src/tea_tasting/datasets.py index 2678312..a4bbe51 100644 --- a/src/tea_tasting/datasets.py +++ b/src/tea_tasting/datasets.py @@ -17,9 +17,14 @@ import numpy.typing as npt try: - from pandas import DataFrame + from pandas import DataFrame as PandasDataFrame except ImportError: - from typing import Any as DataFrame + from typing import Any as PandasDataFrame + + try: + from polars import DataFrame as PolarsDataFrame + except ImportError: + from typing import Any as PolarsDataFrame @overload @@ -35,7 +40,7 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - to_pandas: Literal[False] = False, + result_type: Literal["arrow"] = "arrow", ) -> pa.Table: ... @@ -52,8 +57,25 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - to_pandas: Literal[True] = True, -) -> DataFrame: + result_type: Literal["pandas"] = "pandas", +) -> PandasDataFrame: + ... + +@overload +def make_users_data( + *, + covariates: bool = False, + seed: int | np.random.Generator | np.random.SeedSequence | None = None, + n_users: int = 4000, + ratio: float | int = 1, + sessions_uplift: float | int = 0.0, + orders_uplift: float = 0.1, + revenue_uplift: float = 0.1, + avg_sessions: float | int = 2, + avg_orders_per_session: float = 0.25, + avg_revenue_per_order: float | int = 10, + result_type: Literal["polars"] = "polars", +) -> PolarsDataFrame: ... def make_users_data( @@ -68,8 +90,8 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - to_pandas: bool = False, -) -> pa.Table | DataFrame: + result_type: Literal["arrow", "pandas", "polars"] = "arrow", +) -> pa.Table | PandasDataFrame | PolarsDataFrame: """Generate simulated data for A/B testing scenarios. Data mimics what you might encounter in an A/B test for an online store, @@ -103,8 +125,12 @@ def make_users_data( avg_orders_per_session: Average number of orders per session. Should be less than `1`. avg_revenue_per_order: Average revenue per order. - to_pandas: If set to `True`, returns a Pandas DataFrame; otherwise, - returns a PyArrow Table. + result_type: Result type. + + Result types: + - `"arrow"`: PyArrow Table. + - `"pandas"`: Pandas DataFrame. + - `"polars"`: Polars DataFrame. Returns: Simulated data for A/B testing scenarios. @@ -164,7 +190,7 @@ def make_users_data( avg_sessions=avg_sessions, avg_orders_per_session=avg_orders_per_session, avg_revenue_per_order=avg_revenue_per_order, - to_pandas=to_pandas, + result_type=result_type, explode_sessions=False, ) @@ -182,7 +208,7 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - to_pandas: Literal[False] = False, + result_type: Literal["arrow"] = "arrow", ) -> pa.Table: ... @@ -199,10 +225,11 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - to_pandas: Literal[True] = True, -) -> DataFrame: + result_type: Literal["pandas"] = "pandas", +) -> PandasDataFrame: ... +@overload def make_sessions_data( *, covariates: bool = False, @@ -215,8 +242,24 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - to_pandas: bool = False, -) -> pa.Table | DataFrame: + result_type: Literal["polars"] = "polars", +) -> PolarsDataFrame: + ... + +def make_sessions_data( + *, + covariates: bool = False, + seed: int | np.random.Generator | np.random.SeedSequence | None = None, + n_users: int = 4000, + ratio: float | int = 1, + sessions_uplift: float | int = 0.0, + orders_uplift: float = 0.1, + revenue_uplift: float = 0.1, + avg_sessions: float | int = 2, + avg_orders_per_session: float = 0.25, + avg_revenue_per_order: float | int = 10, + result_type: Literal["arrow", "pandas", "polars"] = "arrow", +) -> pa.Table | PandasDataFrame | PolarsDataFrame: """Generate simulated user data for A/B testing scenarios. Data mimics what you might encounter in an A/B test for an online store, @@ -250,8 +293,12 @@ def make_sessions_data( avg_orders_per_session: Average number of orders per session. Should be less than `1`. avg_revenue_per_order: Average revenue per order. - to_pandas: If set to `True`, returns a Pandas DataFrame; otherwise, - returns a PyArrow Table. + result_type: Result type. + + Result types: + - `"arrow"`: PyArrow Table. + - `"pandas"`: Pandas DataFrame. + - `"polars"`: Polars DataFrame. Returns: Simulated data for A/B testing scenarios. @@ -311,7 +358,7 @@ def make_sessions_data( avg_sessions=avg_sessions, avg_orders_per_session=avg_orders_per_session, avg_revenue_per_order=avg_revenue_per_order, - to_pandas=to_pandas, + result_type=result_type, explode_sessions=True, ) @@ -328,9 +375,9 @@ def _make_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, + result_type: Literal["arrow", "pandas", "polars"] = "arrow", explode_sessions: bool = False, - to_pandas: bool = False, -) -> pa.Table | DataFrame: +) -> pa.Table | PandasDataFrame | PolarsDataFrame: _check_params( n_users=n_users, ratio=ratio, @@ -421,9 +468,12 @@ def _make_data( "revenue_covariate": revenue_covariate, } - if to_pandas: + if result_type == "pandas": import pandas as pd return pd.DataFrame(data) + if result_type == "polars": + import polars as pl + return pl.DataFrame(data) return pa.table(data) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 657bde2..f09b73f 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -2,6 +2,7 @@ from __future__ import annotations import pandas as pd +import polars as pl import pyarrow as pa import pyarrow.compute as pc @@ -28,12 +29,21 @@ def test_make_users_data_default(): def test_make_users_data_pandas(): n_users = 100 data = tea_tasting.datasets.make_users_data( - seed=42, n_users=n_users, to_pandas=True) + seed=42, n_users=n_users, result_type="pandas") assert isinstance(data, pd.DataFrame) assert data.columns.to_list() == [ "user", "variant", "sessions", "orders", "revenue"] assert data.shape[0] == n_users +def test_make_users_data_polars(): + n_users = 100 + data = tea_tasting.datasets.make_users_data( + seed=42, n_users=n_users, result_type="polars") + assert isinstance(data, pl.DataFrame) + assert data.columns == [ + "user", "variant", "sessions", "orders", "revenue"] + assert data.shape[0] == n_users + def test_make_users_data_covariates(): n_users = 100 @@ -78,12 +88,21 @@ def test_make_sessions_data_default(): def test_make_sessions_data_pandas(): n_users = 100 data = tea_tasting.datasets.make_sessions_data( - seed=42, n_users=n_users, to_pandas=True) + seed=42, n_users=n_users, result_type="pandas") assert isinstance(data, pd.DataFrame) assert data.columns.to_list() == [ "user", "variant", "sessions", "orders", "revenue"] assert data.shape[0] > n_users +def test_make_sessions_data_polars(): + n_users = 100 + data = tea_tasting.datasets.make_sessions_data( + seed=42, n_users=n_users, result_type="polars") + assert isinstance(data, pl.DataFrame) + assert data.columns == [ + "user", "variant", "sessions", "orders", "revenue"] + assert data.shape[0] > n_users + def test_make_sessions_data_covariates(): n_users = 100 From b4a65ffca3e555c2519793a9558e77b8fd0abc44 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 21:43:03 +0300 Subject: [PATCH 22/29] Use pyarrow table in tests by default --- tests/test_aggr.py | 103 +++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/tests/test_aggr.py b/tests/test_aggr.py index c2157e5..1e19f60 100644 --- a/tests/test_aggr.py +++ b/tests/test_aggr.py @@ -1,9 +1,13 @@ +# pyright: reportAttributeAccessIssue=false from __future__ import annotations from typing import TYPE_CHECKING import ibis +import numpy as np import polars as pl +import pyarrow as pa +import pyarrow.compute as pc import pytest import tea_tasting.aggr @@ -13,7 +17,6 @@ if TYPE_CHECKING: import ibis.expr.types # noqa: TC004 import pandas as pd - import pyarrow as pa Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame @@ -68,32 +71,51 @@ def data(request: pytest.FixtureRequest) -> Frame: @pytest.fixture -def correct_aggr(data_pandas: pd.DataFrame) -> tea_tasting.aggr.Aggregates: +def correct_aggr(data_arrow: pa.Table) -> tea_tasting.aggr.Aggregates: return tea_tasting.aggr.Aggregates( - count_=len(data_pandas), + count_=data_arrow.num_rows, mean_={ - "sessions": data_pandas["sessions"].mean(), - "orders": data_pandas["orders"].mean(), - }, # type: ignore + "sessions": pc.mean(data_arrow["sessions"]).as_py(), + "orders": pc.mean(data_arrow["orders"]).as_py(), + }, var_={ - "sessions": data_pandas["sessions"].var(), - "orders": data_pandas["orders"].var(), - }, # type: ignore + "sessions": pc.variance(data_arrow["sessions"], ddof=1).as_py(), + "orders": pc.variance(data_arrow["orders"], ddof=1).as_py(), + }, cov_={ - ("orders", "sessions"): data_pandas["sessions"].cov(data_pandas["orders"])}, # type: ignore + ("orders", "sessions"): np.cov( + data_arrow["sessions"].combine_chunks().to_numpy(zero_copy_only=False), + data_arrow["orders"].combine_chunks().to_numpy(zero_copy_only=False), + ddof=1, + )[0, 1], + }, ) @pytest.fixture -def correct_aggrs(data_pandas: pd.DataFrame) -> dict[int, tea_tasting.aggr.Aggregates]: - return { - v: tea_tasting.aggr.Aggregates( - count_=len(d), - mean_={"sessions": d["sessions"].mean(), "orders": d["orders"].mean()}, # type: ignore - var_={"sessions": d["sessions"].var(), "orders": d["orders"].var()}, # type: ignore - cov_={("orders", "sessions"): d["sessions"].cov(d["orders"])}, # type: ignore - ) - for v, d in data_pandas.groupby("variant") - } +def correct_aggrs(data_arrow: pa.Table) -> dict[int, tea_tasting.aggr.Aggregates]: + variant_col = data_arrow["variant"] + aggrs = {} + for var in variant_col.unique().to_pylist(): + var_data = data_arrow.filter(pc.equal(variant_col, pa.scalar(var))) + aggrs |= {var: tea_tasting.aggr.Aggregates( + count_=var_data.num_rows, + mean_={ + "sessions": pc.mean(var_data["sessions"]).as_py(), + "orders": pc.mean(var_data["orders"]).as_py(), + }, + var_={ + "sessions": pc.variance(var_data["sessions"], ddof=1).as_py(), + "orders": pc.variance(var_data["orders"], ddof=1).as_py(), + }, + cov_={ + ("orders", "sessions"): np.cov( + var_data["sessions"].combine_chunks().to_numpy(zero_copy_only=False), + var_data["orders"].combine_chunks().to_numpy(zero_copy_only=False), + ddof=1, + )[0, 1], + }, + )} + return aggrs def test_aggregates_init(aggr: tea_tasting.aggr.Aggregates): @@ -133,34 +155,15 @@ def test_aggregates_ratio_cov(): ) assert aggr.ratio_cov("a", "b", "c", "d") == pytest.approx(-0.0146938775510204) -def test_aggregates_add(data_pandas: pd.DataFrame): - aggr = tea_tasting.aggr.Aggregates( - count_=len(data_pandas), - mean_={ - "sessions": data_pandas["sessions"].mean(), - "orders": data_pandas["orders"].mean(), - }, # type: ignore - var_={ - "sessions": data_pandas["sessions"].var(), - "orders": data_pandas["orders"].var(), - }, # type: ignore - cov_={ - ("sessions", "orders"): data_pandas["sessions"].cov(data_pandas["orders"])}, # type: ignore - ) - aggrs = tuple( - tea_tasting.aggr.Aggregates( - count_=len(d), - mean_={"sessions": d["sessions"].mean(), "orders": d["orders"].mean()}, # type: ignore - var_={"sessions": d["sessions"].var(), "orders": d["orders"].var()}, # type: ignore - cov_={("sessions", "orders"): d["sessions"].cov(d["orders"])}, # type: ignore - ) - for _, d in data_pandas.groupby("variant") - ) - aggrs_add = aggrs[0] + aggrs[1] - assert aggrs_add.count_ == pytest.approx(aggr.count_) - assert aggrs_add.mean_ == pytest.approx(aggr.mean_) - assert aggrs_add.var_ == pytest.approx(aggr.var_) - assert aggrs_add.cov_ == pytest.approx(aggr.cov_) +def test_aggregates_add( + correct_aggr: tea_tasting.aggr.Aggregates, + correct_aggrs: dict[int, tea_tasting.aggr.Aggregates], +): + aggrs_add = correct_aggrs[0] + correct_aggrs[1] + assert aggrs_add.count_ == pytest.approx(correct_aggr.count_) + assert aggrs_add.mean_ == pytest.approx(correct_aggr.mean_) + assert aggrs_add.var_ == pytest.approx(correct_aggr.var_) + assert aggrs_add.cov_ == pytest.approx(correct_aggr.cov_) def test_read_aggregates_groups( @@ -198,9 +201,9 @@ def test_read_aggregates_no_groups( assert aggr.var_ == pytest.approx(correct_aggr.var_) assert aggr.cov_ == pytest.approx(correct_aggr.cov_) -def test_read_aggregates_no_count(data_pandas: pd.DataFrame): +def test_read_aggregates_no_count(data_arrow: pa.Table): aggr = tea_tasting.aggr.read_aggregates( - data_pandas, + data_arrow, group_col=None, has_count=False, mean_cols=("sessions", "orders"), From 938405014307ff64674bd9c1fb095a5694372651 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 21:58:37 +0300 Subject: [PATCH 23/29] Use arrow in tests by default --- tests/test_experiment.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 51b9056..050e4db 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -1,3 +1,4 @@ +# pyright: reportAttributeAccessIssue=false from __future__ import annotations from typing import TYPE_CHECKING, Any, NamedTuple, TypedDict @@ -7,6 +8,7 @@ import narwhals as nw import pandas as pd import polars as pl +import pyarrow as pa import pyarrow.compute as pc import pytest @@ -22,7 +24,6 @@ from typing import Literal import narwhals.typing # noqa: TC004 - import pyarrow as pa Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame @@ -259,24 +260,29 @@ def data(request: pytest.FixtureRequest) -> Frame: return request.getfixturevalue(request.param) @pytest.fixture -def data_pandas_multi(data_pandas: pd.DataFrame) -> pd.DataFrame: - return pd.concat(( - data_pandas, - data_pandas.query("variant==1").assign(variant=2), +def data_arrow_multi(data_arrow: pa.Table) -> pa.Table: + data2 = data_arrow.filter(pc.equal(data_arrow["variant"], pa.scalar(1))) + return pa.concat_tables(( + data_arrow, + data2.set_column( + data_arrow.schema.get_field_index("variant"), + "variant", + pa.array([2] * data2.num_rows), + ), )) @pytest.fixture def ref_result( - data_pandas: pd.DataFrame, + data_arrow: pa.Table, ) -> tea_tasting.experiment.ExperimentResults: sessions = _Metric("sessions") orders = _MetricAggregated("orders") revenue = _MetricGranular("revenue") return tea_tasting.experiment.ExperimentResult( - avg_sessions=sessions.analyze(data_pandas, 0, 1, "variant"), - avg_orders=orders.analyze(data_pandas, 0, 1, "variant"), - avg_revenue=revenue.analyze(data_pandas, 0, 1, "variant"), # type: ignore + avg_sessions=sessions.analyze(data_arrow, 0, 1, "variant"), + avg_orders=orders.analyze(data_arrow, 0, 1, "variant"), + avg_revenue=revenue.analyze(data_arrow, 0, 1, "variant"), # type: ignore ) @@ -425,7 +431,7 @@ def test_experiment_analyze_gran( avg_revenue=ref_result["avg_revenue"]) def test_experiment_analyze_all_pairs( - data_pandas_multi: pd.DataFrame, + data_arrow_multi: pa.Table, ref_result: tea_tasting.experiment.ExperimentResult, ): experiment = tea_tasting.experiment.Experiment({ @@ -433,22 +439,22 @@ def test_experiment_analyze_all_pairs( "avg_orders": _MetricAggregated("orders"), "avg_revenue": _MetricGranular("revenue"), }) - results = experiment.analyze(data_pandas_multi, all_variants=True) + results = experiment.analyze(data_arrow_multi, all_variants=True) assert set(results.keys()) == {(0, 1), (0, 2), (1, 2)} assert results[0, 1] == ref_result assert results[0, 2] == ref_result -def test_experiment_analyze_all_pairs_raises(data_pandas_multi: pd.DataFrame): +def test_experiment_analyze_all_pairs_raises(data_arrow_multi: pa.Table): experiment = tea_tasting.experiment.Experiment({ "avg_sessions": _Metric("sessions"), "avg_orders": _MetricAggregated("orders"), "avg_revenue": _MetricGranular("revenue"), }) with pytest.raises(ValueError, match="all_variants"): - experiment.analyze(data_pandas_multi) + experiment.analyze(data_arrow_multi) def test_experiment_analyze_two_treatments( - data_pandas_multi: pd.DataFrame, + data_arrow_multi: pa.Table, ref_result: tea_tasting.experiment.ExperimentResult, ): experiment = tea_tasting.experiment.Experiment( @@ -458,19 +464,19 @@ def test_experiment_analyze_two_treatments( "avg_revenue": _MetricGranular("revenue"), }, ) - results = experiment.analyze(data_pandas_multi, control=0, all_variants=True) + results = experiment.analyze(data_arrow_multi, control=0, all_variants=True) assert results == tea_tasting.experiment.ExperimentResults({ (0, 1): ref_result, (0, 2): ref_result, }) -def test_experiment_solve_power(data_pandas: pd.DataFrame): +def test_experiment_solve_power(data_arrow: pa.Table): experiment = tea_tasting.experiment.Experiment( metric=_Metric("sessions"), metric_aggr=_MetricAggregated("orders"), ) - result = experiment.solve_power(data_pandas) + result = experiment.solve_power(data_arrow) assert result == tea_tasting.experiment.ExperimentPowerResult({ "metric": tea_tasting.metrics.MetricPowerResults(( _PowerResult(power=0.8, effect_size=1, rel_effect_size=0.05, n_obs=10_000), From d615a7ef20346c887992c55b1630bdc7eb8429c0 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 21:59:26 +0300 Subject: [PATCH 24/29] Minor fix --- tests/metrics/test_mean.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/metrics/test_mean.py b/tests/metrics/test_mean.py index dabc6cc..0660c8f 100644 --- a/tests/metrics/test_mean.py +++ b/tests/metrics/test_mean.py @@ -43,20 +43,20 @@ def data_aggr(data_arrow: pa.Table) -> dict[Any, tea_tasting.aggr.Aggregates]: ) @pytest.fixture -def power_data_pandas() -> pa.Table: +def power_data_arrow() -> pa.Table: return tea_tasting.datasets.make_users_data( n_users=100, covariates=True, seed=42, sessions_uplift=0, orders_uplift=0, revenue_uplift=0, ) @pytest.fixture -def power_data_aggr(power_data_pandas: pa.Table) -> tea_tasting.aggr.Aggregates: +def power_data_aggr(power_data_arrow: pa.Table) -> tea_tasting.aggr.Aggregates: cols = ( "sessions", "orders", "revenue", "sessions_covariate", "orders_covariate", "revenue_covariate", ) return tea_tasting.aggr.read_aggregates( - power_data_pandas, + power_data_arrow, group_col=None, has_count=True, mean_cols=cols, @@ -229,13 +229,13 @@ def test_ratio_of_means_analyze_ratio_less_use_norm( assert result.statistic == pytest.approx(-0.3573188986307722) -def test_ratio_of_means_solve_power_frame(power_data_pandas: pa.Table): +def test_ratio_of_means_solve_power_frame(power_data_arrow: pa.Table): metric = tea_tasting.metrics.mean.RatioOfMeans( numer="orders", denom="sessions", rel_effect_size=0.1, ) - results = metric.solve_power(power_data_pandas, "power") + results = metric.solve_power(power_data_arrow, "power") assert isinstance(results, tea_tasting.metrics.base.MetricPowerResults) assert len(results) == 1 result = results[0] From 0f0bd4c46c6dbf0a295edd602e3a726dc655667c Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 22:01:47 +0300 Subject: [PATCH 25/29] Use arrow in tests by default --- tests/metrics/test_base.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/metrics/test_base.py b/tests/metrics/test_base.py index 881b46e..5e4df66 100644 --- a/tests/metrics/test_base.py +++ b/tests/metrics/test_base.py @@ -110,22 +110,22 @@ def aggr_cols() -> tea_tasting.metrics.base.AggrCols: @pytest.fixture def correct_aggrs( - data_pandas: pd.DataFrame, + data_arrow: pa.Table, aggr_cols: tea_tasting.metrics.base.AggrCols, ) -> dict[Any, tea_tasting.aggr.Aggregates]: return tea_tasting.aggr.read_aggregates( - data_pandas, + data_arrow, group_col="variant", **aggr_cols._asdict(), ) @pytest.fixture def correct_aggr( - data_pandas: pd.DataFrame, + data_arrow: pa.Table, aggr_cols: tea_tasting.metrics.base.AggrCols, ) -> tea_tasting.aggr.Aggregates: return tea_tasting.aggr.read_aggregates( - data_pandas, + data_arrow, group_col=None, **aggr_cols._asdict(), ) @@ -253,11 +253,11 @@ class PowerResult(NamedTuple): def test_metric_base_aggregated_analyze_frame( aggr_metric: tea_tasting.metrics.base.MetricBaseAggregated[dict[str, Any]], - data_pandas: pd.DataFrame, + data_arrow: pa.Table, correct_aggrs: dict[Any, tea_tasting.aggr.Aggregates], ): aggr_metric.analyze_aggregates = unittest.mock.MagicMock() - aggr_metric.analyze(data_pandas, control=0, treatment=1, variant="variant") + aggr_metric.analyze(data_arrow, control=0, treatment=1, variant="variant") aggr_metric.analyze_aggregates.assert_called_once() kwargs = aggr_metric.analyze_aggregates.call_args.kwargs _compare_aggrs(kwargs["control"], correct_aggrs[0]) @@ -277,11 +277,11 @@ def test_metric_base_aggregated_analyze_aggrs( def test_power_base_aggregated_analyze_frame( aggr_power: tea_tasting.metrics.base.PowerBaseAggregated[Any], - data_pandas: pd.DataFrame, + data_arrow: pa.Table, correct_aggr: tea_tasting.aggr.Aggregates, ): aggr_power.solve_power_from_aggregates = unittest.mock.MagicMock() - aggr_power.solve_power(data_pandas, "effect_size") + aggr_power.solve_power(data_arrow, "effect_size") aggr_power.solve_power_from_aggregates.assert_called_once() kwargs = aggr_power.solve_power_from_aggregates.call_args.kwargs _compare_aggrs(kwargs["data"], correct_aggr) @@ -300,12 +300,12 @@ def test_power_base_aggregated_analyze_aggr( def test_aggregate_by_variants_frame( - data_pandas: pd.DataFrame, + data_arrow: pa.Table, aggr_cols: tea_tasting.metrics.base.AggrCols, correct_aggrs: dict[Any, tea_tasting.aggr.Aggregates], ): aggrs = tea_tasting.metrics.base.aggregate_by_variants( - data_pandas, + data_arrow, aggr_cols=aggr_cols, variant="variant", ) @@ -325,20 +325,20 @@ def test_aggregate_by_variants_aggrs( _compare_aggrs(aggrs[1], correct_aggrs[1]) def test_aggregate_by_variants_raises( - data_pandas: pd.DataFrame, + data_arrow: pa.Table, aggr_cols: tea_tasting.metrics.base.AggrCols, ): with pytest.raises(ValueError, match="variant"): - tea_tasting.metrics.base.aggregate_by_variants(data_pandas, aggr_cols=aggr_cols) + tea_tasting.metrics.base.aggregate_by_variants(data_arrow, aggr_cols=aggr_cols) def test_metric_base_granular_frame( gran_metric: tea_tasting.metrics.base.MetricBaseGranular[dict[str, Any]], - data_pandas: pd.DataFrame, + data_arrow: pa.Table, correct_gran: dict[Any, pa.Table], ): gran_metric.analyze_granular = unittest.mock.MagicMock() - gran_metric.analyze(data_pandas, control=0, treatment=1, variant="variant") + gran_metric.analyze(data_arrow, control=0, treatment=1, variant="variant") gran_metric.analyze_granular.assert_called_once() kwargs = gran_metric.analyze_granular.call_args.kwargs assert kwargs["control"].equals(correct_gran[0]) @@ -382,8 +382,8 @@ def test_read_granular_dict( assert gran[1].equals(correct_gran[1]) def test_read_granular_raises( - data_pandas: pd.DataFrame, + data_arrow: pa.Table, cols: tuple[str, ...], ): with pytest.raises(ValueError, match="variant"): - tea_tasting.metrics.base.read_granular(data_pandas, cols=cols) + tea_tasting.metrics.base.read_granular(data_arrow, cols=cols) From 9426421fbe1df53af3f79638cdebe4bb4e54019f Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 22:19:35 +0300 Subject: [PATCH 26/29] Minor fix --- tests/metrics/test_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/metrics/test_base.py b/tests/metrics/test_base.py index 5e4df66..62264c7 100644 --- a/tests/metrics/test_base.py +++ b/tests/metrics/test_base.py @@ -203,8 +203,8 @@ def cols(self) -> tuple[str, ...]: def analyze_granular( self, - control: pd.DataFrame, # noqa: ARG002 - treatment: pd.DataFrame, # noqa: ARG002 + control: pa.Table, # noqa: ARG002 + treatment: pa.Table, # noqa: ARG002 ) -> dict[str, Any]: return {} @@ -346,7 +346,7 @@ def test_metric_base_granular_frame( def test_metric_base_granular_gran( gran_metric: tea_tasting.metrics.base.MetricBaseGranular[dict[str, Any]], - correct_gran: dict[Any, pd.DataFrame], + correct_gran: dict[Any, pa.Table], ): gran_metric.analyze_granular = unittest.mock.MagicMock() gran_metric.analyze(correct_gran, control=0, treatment=1) From c15a6a8100d7cd66e5672bad379ac43c7e209a62 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sat, 28 Dec 2024 22:19:49 +0300 Subject: [PATCH 27/29] Use arrow in tests by default --- tests/test_experiment.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 050e4db..5b4a7ce 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -5,8 +5,6 @@ import ibis import ibis.expr.types -import narwhals as nw -import pandas as pd import polars as pl import pyarrow as pa import pyarrow.compute as pc @@ -23,7 +21,8 @@ from collections.abc import Callable from typing import Literal - import narwhals.typing # noqa: TC004 + import narwhals.typing + import pandas as pd Frame = ibis.expr.types.Table | pa.Table | pd.DataFrame | pl.LazyFrame @@ -61,20 +60,20 @@ def analyze( treatment: int, variant: str, ) -> _MetricResultTuple: - if not isinstance(data, pd.DataFrame): - if not isinstance(data, ibis.expr.types.Table): - data = nw.from_native(data) - if isinstance(data, nw.LazyFrame): - data = data.collect() - data = data.to_pandas() - - agg_data = data.loc[:, [variant, self.value]].groupby(variant).agg("mean") - contr_mean = agg_data.loc[control, self.value] - treat_mean = agg_data.loc[treatment, self.value] + if not isinstance(data, dict): + data = tea_tasting.aggr.read_aggregates( + data, + variant, + has_count=False, + mean_cols=(self.value,), + var_cols=(), + cov_cols=(), + ) return _MetricResultTuple( - control=contr_mean, # type: ignore - treatment=treat_mean, # type: ignore - effect_size=treat_mean - contr_mean, # type: ignore + control=data[control].mean(self.value), + treatment=data[treatment].mean(self.value), + effect_size=data[treatment].mean(self.value) - + data[control].mean(self.value), ) def solve_power( From f66da07c0623e0307d91a3b1247b6609a50043d8 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sun, 5 Jan 2025 15:10:12 +0300 Subject: [PATCH 28/29] Remove unnecessary dependency --- pdm.lock | 37 ++----------------------------------- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/pdm.lock b/pdm.lock index b6bd1c4..a902674 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "docs", "lint", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:8e11adabfb8f2725d292c52f63b33796d86fddb19d91c05f7a4e5e1468d839f1" +content_hash = "sha256:1bf2f7048c02616dda8afd21e347f6bca2d4964c7f9632743f29227c6c681dc2" [[metadata.targets]] requires_python = ">=3.10" @@ -422,20 +422,6 @@ files = [ {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] -[[package]] -name = "markdown-callouts" -version = "0.4.0" -requires_python = ">=3.8" -summary = "Markdown extension: a classier syntax for admonitions" -groups = ["docs"] -dependencies = [ - "markdown>=3.3.3", -] -files = [ - {file = "markdown_callouts-0.4.0-py3-none-any.whl", hash = "sha256:ed0da38f29158d93116a0d0c6ecaf9df90b37e0d989b5337d678ee6e6d6550b7"}, - {file = "markdown_callouts-0.4.0.tar.gz", hash = "sha256:7ed2c90486967058a73a547781121983839522d67041ae52c4979616f1b2b746"}, -] - [[package]] name = "markdown-it-py" version = "3.0.0" @@ -630,24 +616,6 @@ files = [ {file = "mkdocstrings-0.27.0.tar.gz", hash = "sha256:16adca6d6b0a1f9e0c07ff0b02ced8e16f228a9d65a37c063ec4c14d7b76a657"}, ] -[[package]] -name = "mkdocstrings-crystal" -version = "0.3.7" -requires_python = ">=3.8" -summary = "Crystal language doc generator for mkdocstrings" -groups = ["docs"] -dependencies = [ - "jinja2>=2.11.2", - "markdown-callouts>=0.1.0", - "markupsafe>=1.1.1", - "mkdocs-autorefs>=0.3.1", - "mkdocstrings>=0.19.0", -] -files = [ - {file = "mkdocstrings_crystal-0.3.7-py3-none-any.whl", hash = "sha256:91f25700a1e13ee5157aa5875441d333830c392d5ace7ef1c2106d9e5b2883b6"}, - {file = "mkdocstrings_crystal-0.3.7.tar.gz", hash = "sha256:6d0b2fc8ef1256aec2cc4ff22a7d5aff6398c574cede10a5941e5aa3590012c7"}, -] - [[package]] name = "mkdocstrings-python" version = "1.10.9" @@ -667,12 +635,11 @@ files = [ [[package]] name = "mkdocstrings" version = "0.27.0" -extras = ["crystal", "python"] +extras = ["python"] requires_python = ">=3.9" summary = "Automatic documentation from sources, for MkDocs." groups = ["docs"] dependencies = [ - "mkdocstrings-crystal>=0.3.4", "mkdocstrings-python>=0.5.2", "mkdocstrings==0.27.0", ] diff --git a/pyproject.toml b/pyproject.toml index bb5baf8..dee3190 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ build-backend = "pdm.backend" package-dir = "src" [tool.pdm.dev-dependencies] -docs = ["mkdocs-material", "mkdocstrings[crystal,python]"] +docs = ["mkdocs-material", "mkdocstrings[python]"] lint = ["pyright", "ruff"] test = ["coverage[toml]", "ibis-framework[duckdb,sqlite]", "pandas", "polars", "pytest"] From 64b43d6162d8e3dd32ab664082f4e73ebbc82934 Mon Sep 17 00:00:00 2001 From: Evgeny Ivanov Date: Sun, 5 Jan 2025 18:55:32 +0300 Subject: [PATCH 29/29] Update docs and readme --- README.md | 7 +- docs/api/config.md | 2 - docs/api/multiplicity.md | 2 - docs/custom-metrics.md | 42 +++--- docs/data-backends.md | 70 +++++---- docs/index.md | 7 +- docs/multiple-testing.md | 20 ++- docs/power-analysis.md | 12 +- docs/user-guide.md | 84 ++++++++--- mkdocs.yml | 6 - src/tea_tasting/aggr.py | 2 +- src/tea_tasting/config.py | 16 +- src/tea_tasting/datasets.py | 206 +++++++++++++++++++------- src/tea_tasting/experiment.py | 2 +- src/tea_tasting/metrics/proportion.py | 4 +- src/tea_tasting/multiplicity.py | 28 +++- tests/test_datasets.py | 8 +- 17 files changed, 353 insertions(+), 165 deletions(-) diff --git a/README.md b/README.md index 0948ecc..24a4770 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,11 @@ **tea-tasting** is a Python package for the statistical analysis of A/B tests featuring: -- Student's t-test, Z-test, Bootstrap, and quantile metrics out of the box. +- Student's t-test, Z-test, bootstrap, and quantile metrics out of the box. - Extensible API: define and use statistical tests of your choice. - [Delta method](https://alexdeng.github.io/public/files/kdd2018-dm.pdf) for ratio metrics. -- Variance reduction with [CUPED](https://exp-platform.com/Documents/2013-02-CUPED-ImprovingSensitivityOfControlledExperiments.pdf)/[CUPAC](https://doordash.engineering/2020/06/08/improving-experimental-power-through-control-using-predictions-as-covariate-cupac/) (also in combination with the delta method for ratio metrics). -- Confidence intervals for both absolute and percentage change. +- Variance reduction using [CUPED](https://exp-platform.com/Documents/2013-02-CUPED-ImprovingSensitivityOfControlledExperiments.pdf)/[CUPAC](https://doordash.engineering/2020/06/08/improving-experimental-power-through-control-using-predictions-as-covariate-cupac/) (which can also be combined with the delta method for ratio metrics). +- Confidence intervals for both absolute and percentage changes. - Sample ratio mismatch check. - Power analysis. - Multiple hypothesis testing (family-wise error rate and false discovery rate). @@ -56,7 +56,6 @@ Learn more in the detailed [user guide](https://tea-tasting.e10v.me/user-guide/) ## Roadmap -- Switch from Pandas DataFrames to PyArrow Tables for internal data. Make Pandas dependency optional. - A/A tests and simulations. - More statistical tests: - Asymptotic and exact tests for frequency data. diff --git a/docs/api/config.md b/docs/api/config.md index 949861d..5909e00 100644 --- a/docs/api/config.md +++ b/docs/api/config.md @@ -1,3 +1 @@ ::: tea_tasting.config - options: - members_order: source diff --git a/docs/api/multiplicity.md b/docs/api/multiplicity.md index 0042091..4217298 100644 --- a/docs/api/multiplicity.md +++ b/docs/api/multiplicity.md @@ -1,3 +1 @@ ::: tea_tasting.multiplicity - options: - members_order: source diff --git a/docs/custom-metrics.md b/docs/custom-metrics.md index 10641a1..4a97a0f 100644 --- a/docs/custom-metrics.md +++ b/docs/custom-metrics.md @@ -2,12 +2,12 @@ ## Intro -**tea-tasting** supports Student's t-test, Z-test, and [some other statistical tests](api/metrics/index.md) out of the box. However, you might want to analyze an experiment using other statistical criteria. In this case you can define a custom metric with statistical test of your choice. +**tea-tasting** supports Student's t-test, Z-test, and [some other statistical tests](api/metrics/index.md) out of the box. However, you might want to analyze an experiment using other statistical criteria. In this case, you can define a custom metric with a statistical test of your choice. In **tea-tasting**, there are two types of metrics: -- Metrics that require only aggregated statistics for analysis. -- Metrics that require granular data for analysis. +- Metrics that require only aggregated statistics for the analysis. +- Metrics that require granular data for the analysis. This guide explains how to define a custom metric for each type. @@ -17,7 +17,7 @@ First, let's import all the required modules and prepare the data: from typing import Literal, NamedTuple import numpy as np -import pandas as pd +import pyarrow as pa import scipy.stats import tea_tasting as tt import tea_tasting.aggr @@ -26,7 +26,7 @@ import tea_tasting.metrics import tea_tasting.utils -data = tt.make_users_data(seed=42) +data = tt.make_users_data(seed=42, return_type="pandas") data["has_order"] = data.orders.gt(0).astype(int) print(data) #> user variant sessions orders revenue has_order @@ -63,7 +63,7 @@ class ProportionResult(NamedTuple): statistic: float ``` -The second step is defining the metric class itself. Metric based on aggregated statistics should be a subclass of [`MetricBaseAggregated`](api/metrics/base.md#tea_tasting.metrics.base.MetricBaseAggregated). `MetricBaseAggregated` is a generic class with the result class as a type variable. +The second step is defining the metric class itself. A metric based on aggregated statistics should be a subclass of [`MetricBaseAggregated`](api/metrics/base.md#tea_tasting.metrics.base.MetricBaseAggregated). `MetricBaseAggregated` is a generic class with the result class as a type variable. The metric should have the following methods and properties defined: @@ -119,15 +119,15 @@ class Proportion(tea_tasting.metrics.MetricBaseAggregated[ProportionResult]): ) ``` -Method `__init__` save metric parameters to be used in analysis. You can use utility functions [`check_scalar`](api/utils.md#tea_tasting.utils.check_scalar) and [`auto_check`](api/utils.md#tea_tasting.utils.auto_check) to check parameter values. +Method `__init__` saves metric parameters to be used in the analysis. You can use utility functions [`check_scalar`](api/utils.md#tea_tasting.utils.check_scalar) and [`auto_check`](api/utils.md#tea_tasting.utils.auto_check) to check parameter values. Property `aggr_cols` returns an instance of [`AggrCols`](api/metrics/base.md#tea_tasting.metrics.base.AggrCols). Analysis of proportion requires the number of rows (`has_count=True`) and the average value for the column of interest (`mean_cols=(self.column,)`) for each variant. Method `analyze_aggregates` accepts two parameters: `control` and `treatment` data as instances of class [`Aggregates`](api/aggr.md#tea_tasting.aggr.Aggregates). They contain values for statistics and columns specified in `aggr_cols`. -Method `analyze_aggregates` returns an instance of `ProportionResult`, defined earlier, with analysis result. +Method `analyze_aggregates` returns an instance of `ProportionResult`, defined earlier, with the analysis result. -Now we can analyze the proportion of users who created at least one order during the experiment. For comparison, let's also add a metric that performs Z-test on the same column. +Now we can analyze the proportion of users who created at least one order during the experiment. For comparison, let's also add a metric that performs a Z-test on the same column. ```python experiment_prop = tt.Experiment( @@ -142,7 +142,7 @@ print(experiment_prop.analyze(data)) ## Metrics based on granular data -Now let's define a metric that performs the Mann-Whitney U test. While it's possible to use the aggregated sum of ranks in the test, this example will use granular data for analysis. +Now let's define a metric that performs the Mann-Whitney U test. While it's possible to use the aggregated sum of ranks for the test, this example uses granular data for analysis. The result class: @@ -152,13 +152,13 @@ class MannWhitneyUResult(NamedTuple): statistic: float ``` -Metric that analyses granular data should be a subclass of [`MetricBaseGranular`](api/metrics/base.md#tea_tasting.metrics.base.MetricBaseGranular). `MetricBaseGranular` is a generic class with the result class as a type variable. +A metric that analyzes granular data should be a subclass of [`MetricBaseGranular`](api/metrics/base.md#tea_tasting.metrics.base.MetricBaseGranular). `MetricBaseGranular` is a generic class with the result class as a type variable. Metric should have the following methods and properties defined: - Method `__init__` checks and saves metric parameters. - Property `cols` returns columns to be fetched for an analysis. -- Method `analyze_dataframes` analyzes the metric using granular data. +- Method `analyze_granular` analyzes the metric using granular data. ```python class MannWhitneyU(tea_tasting.metrics.MetricBaseGranular[MannWhitneyUResult]): @@ -181,14 +181,14 @@ class MannWhitneyU(tea_tasting.metrics.MetricBaseGranular[MannWhitneyUResult]): def cols(self) -> tuple[str]: return (self.column,) - def analyze_dataframes( + def analyze_granular( self, - control: pd.DataFrame, - treatment: pd.DataFrame, + control: pa.Table, + treatment: pa.Table, ) -> MannWhitneyUResult: res = scipy.stats.mannwhitneyu( - treatment[self.column], - control[self.column], + treatment[self.column].combine_chunks().to_numpy(zero_copy_only=False), + control[self.column].combine_chunks().to_numpy(zero_copy_only=False), use_continuity=self.correction, alternative=self.alternative, ) @@ -200,9 +200,9 @@ class MannWhitneyU(tea_tasting.metrics.MetricBaseGranular[MannWhitneyUResult]): Property `cols` should return a sequence of strings. -Method `analyze_dataframes` accepts two parameters: control and treatment data as Pandas DataFrames. Even with [data backend](data-backends.md) different from Pandas, **tea-tasting** will retrieve the data and transform into a Pandas DataFrame. +Method `analyze_granular` accepts two parameters: control and treatment data as PyArrow Tables. Even with [data backend](data-backends.md) different from PyArrow, **tea-tasting** will retrieve the data and transform into a PyArrow Table. -Method `analyze_dataframes` returns an instance of `MannWhitneyUResult`, defined earlier, with analysis result. +Method `analyze_granular` returns an instance of `MannWhitneyUResult`, defined earlier, with analysis result. Now we can perform the Mann-Whitney U test: @@ -237,7 +237,7 @@ print(experiment.analyze(data)) #> mwu_revenue - - - [-, -] 0.0300 ``` -In this case, **tea-tasting** perform two queries on experimental data: +In this case, **tea-tasting** performs two queries on the experimental data: - With aggregated statistics required for analysis of metrics of type `MetricBaseAggregated`. - With detailed data with columns required for analysis of metrics of type `MetricBaseGranular`. @@ -249,4 +249,4 @@ Follow these recommendations when defining custom metrics: - Use parameter and attribute names consistent with the ones that are already defined in **tea-tasting**. For example, use `pvalue` instead of `p_value` or `correction` instead of `use_continuity`. - End confidence interval boundary names with `"_ci_lower"` and `"_ci_upper"`. - During initialization, save parameter values in metric attributes using the same names. For example, use `self.correction = correction` instead of `self.use_continuity = correction`. -- Use globals settings as default values for standard parameters, such as `alternative` or `confidence_level`. See the [reference](api/config.md#tea_tasting.config.config_context) for the full list of standard parameters. You can also define and use your own global parameters. +- Use global settings as default values for standard parameters, such as `alternative` or `confidence_level`. See the [reference](api/config.md#tea_tasting.config.config_context) for the full list of standard parameters. You can also define and use your own global parameters. diff --git a/docs/data-backends.md b/docs/data-backends.md index 1487b6c..39bfcef 100644 --- a/docs/data-backends.md +++ b/docs/data-backends.md @@ -27,6 +27,7 @@ First, let's prepare a demo database: ```python import ibis +import polars as pl import tea_tasting as tt @@ -35,7 +36,7 @@ con = ibis.duckdb.connect() con.create_table("users_data", users_data) #> DatabaseTable: memory.main.users_data #> user int64 -#> variant uint8 +#> variant int64 #> sessions int64 #> orders int64 #> revenue float64 @@ -51,7 +52,7 @@ See the [Ibis documentation on how to create connections](https://ibis-project.o ## Querying experimental data -Method `con.create_table` in the example above returns an instance of Ibis Table which already can be used in the analysis of the experiment. But let's see how to use an SQL query to create Ibis Table: +Method `con.create_table` in the example above returns an Ibis Table which already can be used in the analysis of the experiment. But let's see how to use an SQL query to create an Ibis Table: ```python data = con.sql("select * from users_data") @@ -61,30 +62,39 @@ print(data) #> select * from users_data #> schema: #> user int64 -#> variant uint8 +#> variant int64 #> sessions int64 #> orders int64 #> revenue float64 ``` -It's a very simple query. In real world, you might need to use joins, aggregations, and CTEs to get the data. You can define any SQL query supported by your data backend and use it to create Ibis Table. +It's a very simple query. In the real world, you might need to use joins, aggregations, and CTEs to get the data. You can define any SQL query supported by your data backend and use it to create Ibis Table. Keep in mind that **tea-tasting** assumes that: - Data is grouped by randomization units, such as individual users. -- There is a column indicating variant of the A/B test (typically labeled as A, B, etc.). +- There is a column indicating the variant of the A/B test (typically labeled as A, B, etc.). - All necessary columns for metric calculations (like the number of orders, revenue, etc.) are included in the table. Ibis Table is a lazy object. It doesn't fetch the data when created. You can use Ibis DataFrame API to query the table and fetch the result: ```python -print(data.head(5).to_pandas()) -#> user variant sessions orders revenue -#> 0 0 1 2 1 9.166147 -#> 1 1 0 2 1 6.434079 -#> 2 2 1 2 1 7.943873 -#> 3 3 1 2 1 15.928675 -#> 4 4 0 1 1 7.136917 +with pl.Config( + float_precision=5, + tbl_cell_alignment="RIGHT", + tbl_formatting="NOTHING", + trim_decimal_zeros=False, +): + print(data.head(5).to_polars()) +#> shape: (5, 5) +#> user variant sessions orders revenue +#> --- --- --- --- --- +#> i64 i64 i64 i64 f64 +#> 0 1 2 1 9.16615 +#> 1 0 2 1 6.43408 +#> 2 1 2 1 7.94387 +#> 3 1 2 1 15.92867 +#> 4 0 1 1 7.13692 ``` ## Ibis example @@ -104,7 +114,7 @@ print(aggr_data) #> select * from users_data #> schema: #> user int64 -#> variant uint8 +#> variant int64 #> sessions int64 #> orders int64 #> revenue float64 @@ -122,10 +132,19 @@ print(aggr_data) `aggr_data` is another Ibis Table defined as a query over the previously defined `data`. Let's fetch the result: ```python -print(aggr_data.to_pandas()) -#> variant sessions_per_user orders_per_session orders_per_user revenue_per_user -#> 0 0 1.996045 0.265726 0.530400 5.241079 -#> 1 1 1.982802 0.289031 0.573091 5.730132 +with pl.Config( + float_precision=5, + tbl_cell_alignment="RIGHT", + tbl_formatting="NOTHING", + trim_decimal_zeros=False, +): + print(aggr_data.to_polars()) +#> shape: (2, 5) +#> variant sessions_per_user orders_per_session orders_per_user revenue_per_user +#> --- --- --- --- --- +#> i64 f64 f64 f64 f64 +#> 0 1.99605 0.26573 0.53040 5.24108 +#> 1 1.98280 0.28903 0.57309 5.73013 ``` Internally, Ibis compiles a Table to an SQL query supported by the backend: @@ -151,7 +170,7 @@ See [Ibis documentation](https://ibis-project.org/tutorials/getting_started) for ## Experiment analysis -The example above shows how to query the metric averages. But for statistical inference it's not enough. For example, Student's t-test and Z-test also require number of rows and variance. And analysis of ratio metrics and variance reduction with CUPED require covariances. +The example above shows how to query the metric averages. But for statistical inference, it's not enough. For example, Student's t-test and Z-test also require number of rows and variance. Additionally, analysis of ratio metrics and variance reduction with CUPED requires covariances. Querying all the required statistics manually can be a daunting and error-prone task. But don't worry—**tea-tasting** does this work for you. You just need to specify the metrics: @@ -171,9 +190,9 @@ print(result) #> revenue_per_user 5.24 5.73 9.3% [-2.4%, 22%] 0.123 ``` -In the example above, **tea-tasting** fetches all the required statistics with a single query and then uses them to analyse the experiment. +In the example above, **tea-tasting** fetches all the required statistics with a single query and then uses them to analyze the experiment. -Some statistical methods, like Bootstrap, require granular data for the analysis. In this case, **tea-tasting** fetches the detailed data as well. +Some statistical methods, like bootstrap, require granular data for analysis. In this case, **tea-tasting** fetches the detailed data as well. ## Example with CUPED @@ -184,7 +203,7 @@ users_data_with_cov = tt.make_users_data(seed=42, covariates=True) con.create_table("users_data_with_cov", users_data_with_cov) #> DatabaseTable: memory.main.users_data_with_cov #> user int64 -#> variant uint8 +#> variant int64 #> sessions int64 #> orders int64 #> revenue float64 @@ -215,14 +234,11 @@ print(result_with_cov) ## Polars example -An example of analysis using a Polars DataFrame as input data: +Here’s an example of how to analyze data using a Polars DataFrame: ```python -import polars as pl - - -polars_data = pl.from_pandas(users_data) -print(experiment.analyze(polars_data)) +data_polars = pl.from_arrow(users_data) +print(experiment.analyze(data_polars)) #> metric control treatment rel_effect_size rel_effect_size_ci pvalue #> sessions_per_user 2.00 1.98 -0.66% [-3.7%, 2.5%] 0.674 #> orders_per_session 0.266 0.289 8.8% [-0.89%, 19%] 0.0762 diff --git a/docs/index.md b/docs/index.md index 0948ecc..24a4770 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,11 +9,11 @@ **tea-tasting** is a Python package for the statistical analysis of A/B tests featuring: -- Student's t-test, Z-test, Bootstrap, and quantile metrics out of the box. +- Student's t-test, Z-test, bootstrap, and quantile metrics out of the box. - Extensible API: define and use statistical tests of your choice. - [Delta method](https://alexdeng.github.io/public/files/kdd2018-dm.pdf) for ratio metrics. -- Variance reduction with [CUPED](https://exp-platform.com/Documents/2013-02-CUPED-ImprovingSensitivityOfControlledExperiments.pdf)/[CUPAC](https://doordash.engineering/2020/06/08/improving-experimental-power-through-control-using-predictions-as-covariate-cupac/) (also in combination with the delta method for ratio metrics). -- Confidence intervals for both absolute and percentage change. +- Variance reduction using [CUPED](https://exp-platform.com/Documents/2013-02-CUPED-ImprovingSensitivityOfControlledExperiments.pdf)/[CUPAC](https://doordash.engineering/2020/06/08/improving-experimental-power-through-control-using-predictions-as-covariate-cupac/) (which can also be combined with the delta method for ratio metrics). +- Confidence intervals for both absolute and percentage changes. - Sample ratio mismatch check. - Power analysis. - Multiple hypothesis testing (family-wise error rate and false discovery rate). @@ -56,7 +56,6 @@ Learn more in the detailed [user guide](https://tea-tasting.e10v.me/user-guide/) ## Roadmap -- Switch from Pandas DataFrames to PyArrow Tables for internal data. Make Pandas dependency optional. - A/A tests and simulations. - More statistical tests: - Asymptotic and exact tests for frequency data. diff --git a/docs/multiple-testing.md b/docs/multiple-testing.md index cf39e6a..ac6b05d 100644 --- a/docs/multiple-testing.md +++ b/docs/multiple-testing.md @@ -13,7 +13,7 @@ The [multiple hypothesis testing problem](https://en.wikipedia.org/wiki/Multiple - Holm's step-down procedure, assuming arbitrary dependence between hypotheses. - Hochberg's step-up procedure, assuming non-negative correlation between hypotheses. -As an example, let's consider an experiment with three variants, a control and two treatments: +As an example, consider an experiment with three variants, a control and two treatments: ```python import pandas as pd @@ -21,8 +21,18 @@ import tea_tasting as tt data = pd.concat(( - tt.make_users_data(seed=42, orders_uplift=0.10, revenue_uplift=0.15), - tt.make_users_data(seed=21, orders_uplift=0.15, revenue_uplift=0.20) + tt.make_users_data( + seed=42, + orders_uplift=0.10, + revenue_uplift=0.15, + return_type="pandas", + ), + tt.make_users_data( + seed=21, + orders_uplift=0.15, + revenue_uplift=0.20, + return_type="pandas", + ) .query("variant==1") .assign(variant=2), )) @@ -66,13 +76,13 @@ print(results) #> (0, 2) revenue_per_user 5.24 6.25 19% [6.6%, 33%] 0.00218 ``` -Suppose only the two metrics `orders_per_user` and `revenue_per_user` are considered as success metrics, while the two other metrics `sessions_per_user` and `orders_per_session` are second-orders diagnostic metrics. +Suppose only the two metrics `orders_per_user` and `revenue_per_user` are considered as success metrics, while the other two metrics `sessions_per_user` and `orders_per_session` are second-order diagnostic metrics. ```python metrics = {"orders_per_user", "revenue_per_user"} ``` -With two treatment variants and two success metrics, there are four hypotheses in total, which increases the probability of false positives (also called "false discoveries"). It's recommended to adjust the p-values or the significance level alpha in this case. Let's explore the correction methods provided by **tea-tasting**. +With two treatment variants and two success metrics, there are four hypotheses in total, which increases the probability of false positives (also called "false discoveries"). It's recommended to adjust the p-values or the significance level (alpha) in this case. Let's explore the correction methods provided by **tea-tasting**. ## False discovery rate diff --git a/docs/power-analysis.md b/docs/power-analysis.md index 467b083..8820307 100644 --- a/docs/power-analysis.md +++ b/docs/power-analysis.md @@ -1,12 +1,12 @@ # Power analysis -In **tea-tasting**, you can analyze statistical power for `Mean` and `RatioOfMeans` metrics. There are three possible options: +In **tea-tasting**, you can analyze the statistical power for `Mean` and `RatioOfMeans` metrics. There are three possible options: - Calculate the effect size, given statistical power and the total number of observations. - Calculate the total number of observations, given statistical power and the effect size. - Calculate statistical power, given the effect size and the total number of observations. -In the following example, **tea-tasting** calculates statistical power given the relative effect size and the number of observations: +In this example, **tea-tasting** calculates statistical power given the relative effect size and the number of observations: ```python import tea_tasting as tt @@ -26,7 +26,7 @@ print(orders_per_session.solve_power(data, "power")) #> 52% 0.0261 10% 4000 ``` -Besides `alternative`, `equal_var`, `use_t`, and covariates (CUPED), the following metric parameters impact the result: +Besides `alternative`, `equal_var`, `use_t`, and covariates (CUPED), the following metric parameters affect the result: - `alpha`: Significance level. - `ratio`: Ratio of the number of observations in the treatment relative to the control. @@ -34,7 +34,7 @@ Besides `alternative`, `equal_var`, `use_t`, and covariates (CUPED), the followi - `effect_size` and `rel_effect_size`: Absolute and relative effect size. Only one of them can be defined. - `n_obs`: Number of observations in the control and in the treatment together. If the number of observations is not set explicitly, it's inferred from the dataset. -You can change default values of `alpha`, `ratio`, `power`, and `n_obs` using the [global settings](user-guide.md#global-settings). +You can change the default values of `alpha`, `ratio`, `power`, and `n_obs` using the [global settings](user-guide.md#global-settings). **tea-tasting** can analyze power for several values of parameters `effect_size`, `rel_effect_size`, or `n_obs`. Example: @@ -75,6 +75,6 @@ print(power_result) #> revenue_per_user 80% 0.345 6.5% 20000 ``` -In the example above, **tea-tasting** calculates the relative and absolute effect size for all metrics for two possible sample size values, `10_000` and `20_000`. +In the example above, **tea-tasting** calculates both the relative and absolute effect size for all metrics for two possible sample size values, `10_000` and `20_000`. -The `solve_power` methods of a [metric](api/metrics/mean.md#tea_tasting.metrics.mean.Mean.solve_power) and of an [experiment](api/experiment.md#tea_tasting.experiment.Experiment.solve_power) return the instances of [`MetricPowerResults`](api/metrics/base.md#tea_tasting.metrics.base.MetricPowerResults) and [`ExperimentPowerResult`](api/experiment.md#tea_tasting.experiment.ExperimentPowerResult) respectively. These result classes provide the serialization methods similar to the experiment result: `to_dicts`, `to_pandas`, `to_pretty`, `to_string`, `to_html`. +The `solve_power` methods of a [metric](api/metrics/mean.md#tea_tasting.metrics.mean.Mean.solve_power) and of an [experiment](api/experiment.md#tea_tasting.experiment.Experiment.solve_power) return the instances of [`MetricPowerResults`](api/metrics/base.md#tea_tasting.metrics.base.MetricPowerResults) and [`ExperimentPowerResult`](api/experiment.md#tea_tasting.experiment.ExperimentPowerResult) respectively. These result classes provide the serialization methods similar to the experiment result: `to_dicts`, `to_arrow`, `to_pandas`, `to_polars`, `to_pretty_dicts`, `to_string`, `to_html`. They are also rendered as HTML tables in IPython, Jupyter, or Marimo. diff --git a/docs/user-guide.md b/docs/user-guide.md index 51b8f75..17a46b7 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -6,6 +6,8 @@ pip install tea-tasting ``` +Install Pandas or Polars to serialize analysis results as a Pandas DataFrame or a Polars DataFrame, respectively. These packages are not installed with **tea-tasting** by default. + ## Basic usage Begin with this simple example to understand the basic functionality: @@ -44,6 +46,26 @@ The [`make_users_data`](api/datasets.md#tea_tasting.datasets.make_users_data) fu - `orders`: The total number of user's orders. - `revenue`: The total revenue generated by the user. +By default, `make_users_data` returns a PyArrow Table: + +```python +print(data) +#> pyarrow.Table +#> user: int64 +#> variant: int64 +#> sessions: int64 +#> orders: int64 +#> revenue: double +#> ---- +#> user: [[0,1,2,3,4,...,3995,3996,3997,3998,3999]] +#> variant: [[1,0,1,1,0,...,0,0,0,0,0]] +#> sessions: [[2,2,2,2,1,...,2,2,3,1,5]] +#> orders: [[1,1,1,1,1,...,0,0,0,0,2]] +#> revenue: [[9.166147128806545,6.4340787057460656,7.943873223822707,15.928674729738708,7.136917019113867,...,0,0,0,0,17.162458516177704]] +``` + +You can control return type using the `return_type` parameter. The other possible output types are Pandas DataFrame and Polars DataFrame. They require Pandas or Polars packages respectively. + **tea-tasting** can process data in the form of an Ibis Table or a DataFrame supported by Narwhals: - [Ibis](https://github.com/ibis-project/ibis) is a DataFrame API to various data backends. It supports many backends including BigQuery, ClickHouse, DuckDB, PostgreSQL, Snowflake, Spark etc. You can write an SQL query, [wrap](https://ibis-project.org/how-to/extending/sql#backend.sql) it as an Ibis Table and pass it to **tea-tasting**. @@ -64,7 +86,7 @@ The [`Experiment`](api/experiment.md#tea_tasting.experiment.Experiment) class de - Using keyword parameters, with metric names as parameter names, and metric definitions as parameter values, as in example above. - Using the first argument `metrics` which accepts metrics in a form of dictionary with metric names as keys and metric definitions as values. -By default, **tea-testing** assumes that the A/B test variant is stored in a column named `"variant"`. You can change it, using the `variant` parameter of the `Experiment` class. +By default, **tea-tasting** assumes that the A/B test variant is stored in a column named `"variant"`. You can change it using the `variant` parameter of the `Experiment` class. Example usage: @@ -101,7 +123,7 @@ Use the following parameters of `Mean` and `RatioOfMeans` to customize the analy Example usage: ```python -experiment = tt.Experiment( +new_experiment = tt.Experiment( sessions_per_user=tt.Mean("sessions", alternative="greater"), orders_per_session=tt.RatioOfMeans("orders", "sessions", confidence_level=0.9), orders_per_user=tt.Mean("orders", equal_var=True), @@ -156,8 +178,10 @@ Fields in result depend on metrics. For `Mean` and `RatioOfMeans`, the [fields i [`ExperimentResult`](api/experiment.md#tea_tasting.experiment.ExperimentResult) provides the following methods to serialize and view the experiment result: - `to_dicts`: Convert the result to a sequence of dictionaries. -- `to_pandas`: Convert the result to a Pandas DataFrame. -- `to_pretty`: Convert the result to a Pandas Dataframe with formatted values (as strings). +- `to_arrow`: Convert the result to a PyArrow Table. +- `to_pandas`: Convert the result to a Pandas DataFrame. Requires Pandas to be installed. +- `to_polars`: Convert the result to a Polars DataFrame. Requires Polars to be installed. +- `to_pretty_dicts`: Convert the result to a sequence of dictionaries with formatted values (as strings). - `to_string`: Convert the result to a string. - `to_html`: Convert the result to HTML. @@ -172,10 +196,13 @@ print(result) #> revenue_per_user 5.24 5.73 9.3% [-2.4%, 22%] 0.123 ``` -By default, methods `to_pretty`, `to_string`, and `to_html` return a predefined list of attributes. This list can be customized: +`ExperimentResult` provides also the `_repr_html_` method and is rendered as HTML table in IPython, Jupyter, or Marimo. + +By default, methods `to_pretty_dicts`, `to_string`, and `to_html` return a predefined list of attributes. This list can be customized: ```python -print(result.to_string(names=( +print(result.to_string(keys=( + "metric", "control", "treatment", "effect_size", @@ -188,8 +215,6 @@ print(result.to_string(names=( #> revenue_per_user 5.24 5.73 0.489 [-0.133, 1.11] ``` -In Jupyter and IPython, the output of the line `result` will be a rendered HTML table. - ## More features ### Variance reduction with CUPED/CUPAC @@ -247,14 +272,18 @@ import tea_tasting as tt experiment = tt.Experiment( + orders_per_user=tt.Mean("orders"), + revenue_per_user=tt.Mean("revenue"), sample_ratio=tt.SampleRatio(), ) data = tt.make_users_data(seed=42) result = experiment.analyze(data) -print(result.to_string(("control", "treatment", "pvalue"))) -#> metric control treatment pvalue -#> sample_ratio 2023 1977 0.477 +print(result) +#> metric control treatment rel_effect_size rel_effect_size_ci pvalue +#> orders_per_user 0.530 0.573 8.0% [-2.0%, 19%] 0.118 +#> revenue_per_user 5.24 5.73 9.3% [-2.4%, 22%] 0.123 +#> sample_ratio 2023 1977 - [-, -] 0.477 ``` By default, `SampleRatio` expects equal number of observations across all variants. To specify a different ratio, use the `ratio` parameter. It accepts two types of values: @@ -292,7 +321,7 @@ Use [`get_config`](api/config.md#tea_tasting.config.get_config) with the option import tea_tasting as tt -tt.get_config("equal_var") +print(tt.get_config("equal_var")) #> False ``` @@ -314,14 +343,18 @@ experiment = tt.Experiment( revenue_per_user=tt.Mean("revenue"), ) -experiment.metrics["orders_per_user"] +print(experiment.metrics["orders_per_user"]) #> Mean(value='orders', covariate=None, alternative='two-sided', -#> confidence_level=0.95, equal_var=True, use_t=False) +#> confidence_level=0.95, equal_var=True, use_t=False, +#> alpha=0.05, ratio=1, power=0.8, effect_size=None, rel_effect_size=None, +#> n_obs=None) ``` Use [`config_context`](api/config.md#tea_tasting.config.config_context) to temporarily set a global option value within a context: ```python +tt.set_config(equal_var=False, use_t=True) + with tt.config_context(equal_var=True, use_t=False): experiment = tt.Experiment( sessions_per_user=tt.Mean("sessions"), @@ -330,9 +363,17 @@ with tt.config_context(equal_var=True, use_t=False): revenue_per_user=tt.Mean("revenue"), ) -experiment.metrics["orders_per_user"] +print(tt.get_config("equal_var")) +#> False + +print(tt.get_config("use_t")) +#> True + +print(experiment.metrics["orders_per_user"]) #> Mean(value='orders', covariate=None, alternative='two-sided', -#> confidence_level=0.95, equal_var=True, use_t=False) +#> confidence_level=0.95, equal_var=True, use_t=False, +#> alpha=0.05, ratio=1, power=0.8, effect_size=None, rel_effect_size=None, +#> n_obs=None) ``` ### More than two variants @@ -342,9 +383,14 @@ In **tea-tasting**, it's possible to analyze experiments with more than two vari Example usage: ```python +import pandas as pd +import tea_tasting as tt + + data = pd.concat(( - tt.make_users_data(seed=42), - tt.make_users_data(seed=21).query("variant==1").assign(variant=2), + tt.make_users_data(seed=42, return_type="pandas"), + tt.make_users_data(seed=21, return_type="pandas") + .query("variant==1").assign(variant=2), )) experiment = tt.Experiment( @@ -403,4 +449,4 @@ print(results[0, 1]) #> revenue_per_user 5.24 5.73 9.3% [-2.4%, 22%] 0.123 ``` -By default, **tea-tasting** does not adjust for multiple hypothesis testing. However, it provides several methods for multiple testing correction. For more details, see the the [guide on multiple hypothesis testing](multiple-testing.md). +By default, **tea-tasting** does not adjust for multiple hypothesis testing. However, it provides several methods for multiple testing correction. For more details, see the [guide on multiple hypothesis testing](multiple-testing.md). diff --git a/mkdocs.yml b/mkdocs.yml index 49b9fb1..4bde419 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -78,12 +78,6 @@ plugins: - search markdown_extensions: - - pymdownx.highlight: - anchor_linenums: true - line_spans: __span - pygments_lang_class: true - - pymdownx.inlinehilite - - pymdownx.snippets - pymdownx.superfences - toc: permalink: "#" diff --git a/src/tea_tasting/aggr.py b/src/tea_tasting/aggr.py index 6588a97..0092352 100644 --- a/src/tea_tasting/aggr.py +++ b/src/tea_tasting/aggr.py @@ -264,7 +264,7 @@ def read_aggregates( var_cols: Sequence[str], cov_cols: Sequence[tuple[str, str]], ) -> dict[Any, Aggregates] | Aggregates: - """Extract aggregated statistics from an Ibis Table or a Pandas DataFrame. + """Extract aggregated statistics. Args: data: Granular data. diff --git a/src/tea_tasting/config.py b/src/tea_tasting/config.py index 7415df2..d73929a 100644 --- a/src/tea_tasting/config.py +++ b/src/tea_tasting/config.py @@ -42,7 +42,7 @@ def get_config(option: str | None = None) -> Any: import tea_tasting as tt - tt.get_config("equal_var") + print(tt.get_config("equal_var")) #> False ``` """ @@ -83,7 +83,7 @@ def set_config( relative to the control. Default is 1. use_t: Defines whether to use the Student's t-distribution (`True`) or the Normal distribution (`False`) by default. Default is `True`. - kwargs: User-defined global parameters. + **kwargs: User-defined global parameters. Alternative hypothesis options: - `"two-sided"`: the means are unequal, @@ -106,9 +106,10 @@ def set_config( revenue_per_user=tt.Mean("revenue"), ) - experiment.metrics["orders_per_user"] + print(experiment.metrics["orders_per_user"]) #> Mean(value='orders', covariate=None, alternative='two-sided', - #> confidence_level=0.95, equal_var=True, use_t=False) + #> confidence_level=0.95, equal_var=True, use_t=False, alpha=0.05, ratio=1, + #> power=0.8, effect_size=None, rel_effect_size=None, n_obs=None) ``` """ params = {k: v for k, v in locals().items() if k != "kwargs"} | kwargs @@ -150,7 +151,7 @@ def config_context( relative to the control. Default is 1. use_t: Defines whether to use the Student's t-distribution (`True`) or the Normal distribution (`False`) by default. Default is `True`. - kwargs: User-defined global parameters. + **kwargs: User-defined global parameters. Alternative hypothesis options: - `"two-sided"`: the means are unequal, @@ -172,9 +173,10 @@ def config_context( revenue_per_user=tt.Mean("revenue"), ) - experiment.metrics["orders_per_user"] + print(experiment.metrics["orders_per_user"]) #> Mean(value='orders', covariate=None, alternative='two-sided', - #> confidence_level=0.95, equal_var=True, use_t=False) + #> confidence_level=0.95, equal_var=True, use_t=False, alpha=0.05, ratio=1, + #> power=0.8, effect_size=None, rel_effect_size=None, n_obs=None) ``` """ new_config = {k: v for k, v in locals().items() if k != "kwargs"} | kwargs diff --git a/src/tea_tasting/datasets.py b/src/tea_tasting/datasets.py index a4bbe51..3a6acbf 100644 --- a/src/tea_tasting/datasets.py +++ b/src/tea_tasting/datasets.py @@ -40,7 +40,7 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["arrow"] = "arrow", + return_type: Literal["arrow"] = "arrow", ) -> pa.Table: ... @@ -57,7 +57,7 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["pandas"] = "pandas", + return_type: Literal["pandas"] = "pandas", ) -> PandasDataFrame: ... @@ -74,7 +74,7 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["polars"] = "polars", + return_type: Literal["polars"] = "polars", ) -> PolarsDataFrame: ... @@ -90,7 +90,7 @@ def make_users_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["arrow", "pandas", "polars"] = "arrow", + return_type: Literal["arrow", "pandas", "polars"] = "arrow", ) -> pa.Table | PandasDataFrame | PolarsDataFrame: """Generate simulated data for A/B testing scenarios. @@ -125,7 +125,7 @@ def make_users_data( avg_orders_per_session: Average number of orders per session. Should be less than `1`. avg_revenue_per_order: Average revenue per order. - result_type: Result type. + return_type: Result type. Result types: - `"arrow"`: PyArrow Table. @@ -141,7 +141,51 @@ def make_users_data( data = tt.make_users_data(seed=42) - data + print(data) + #> pyarrow.Table + #> user: int64 + #> variant: int64 + #> sessions: int64 + #> orders: int64 + #> revenue: double + #> ---- + #> user: [[0,1,2,3,4,...,3995,3996,3997,3998,3999]] + #> variant: [[1,0,1,1,0,...,0,0,0,0,0]] + #> sessions: [[2,2,2,2,1,...,2,2,3,1,5]] + #> orders: [[1,1,1,1,1,...,0,0,0,0,2]] + #> revenue: [[9.166147128806545,6.4340787057460656,7.943873223822707,15.928674729738708,7.136917019113867,...,0,0,0,0,17.162458516177704]] + ``` + + With covariates: + + ```python + data = tt.make_users_data(seed=42, covariates=True) + print(data) + #> pyarrow.Table + #> user: int64 + #> variant: int64 + #> sessions: int64 + #> orders: int64 + #> revenue: double + #> sessions_covariate: int64 + #> orders_covariate: int64 + #> revenue_covariate: double + #> ---- + #> user: [[0,1,2,3,4,...,3995,3996,3997,3998,3999]] + #> variant: [[1,0,1,1,0,...,0,0,0,0,0]] + #> sessions: [[2,2,2,2,1,...,2,2,3,1,5]] + #> orders: [[1,1,1,1,1,...,0,0,0,0,2]] + #> revenue: [[9.166147128806545,6.4340787057460656,7.943873223822707,15.928674729738708,7.136917019113867,...,0,0,0,0,17.162458516177704]] + #> sessions_covariate: [[3,4,4,1,1,...,1,3,2,1,5]] + #> orders_covariate: [[2,1,2,0,1,...,0,1,0,0,0]] + #> revenue_covariate: [[19.191712010123307,2.7707490091913525,22.56842219448677,0,13.683796263730468,...,0,13.517967243105218,0,0,0]] + ``` + + As Pandas DataFrame: + + ```python + data = tt.make_users_data(seed=42, return_type="pandas") + print(data) #> user variant sessions orders revenue #> 0 0 1 2 1 9.166147 #> 1 1 0 2 1 6.434079 @@ -158,25 +202,34 @@ def make_users_data( #> [4000 rows x 5 columns] ``` - With covariates: + As Polars DataFrame: ```python - data = tt.make_users_data(seed=42, covariates=True) - data - #> user variant sessions orders revenue sessions_covariate orders_covariate revenue_covariate - #> 0 0 1 2 1 9.166147 3 2 19.191712 - #> 1 1 0 2 1 6.434079 4 1 2.770749 - #> 2 2 1 2 1 7.943873 4 2 22.568422 - #> 3 3 1 2 1 15.928675 1 0 0.000000 - #> 4 4 0 1 1 7.136917 1 1 13.683796 - #> ... ... ... ... ... ... ... ... ... - #> 3995 3995 0 2 0 0.000000 1 0 0.000000 - #> 3996 3996 0 2 0 0.000000 3 1 13.517967 - #> 3997 3997 0 3 0 0.000000 2 0 0.000000 - #> 3998 3998 0 1 0 0.000000 1 0 0.000000 - #> 3999 3999 0 5 2 17.162459 5 0 0.000000 - #> - #> [4000 rows x 8 columns] + import polars as pl + + data = tt.make_users_data(seed=42, return_type="polars") + with pl.Config( + float_precision=5, + tbl_cell_alignment="RIGHT", + tbl_formatting="NOTHING", + trim_decimal_zeros=False, + ): + print(data) + #> shape: (4_000, 5) + #> user variant sessions orders revenue + #> --- --- --- --- --- + #> i64 i64 i64 i64 f64 + #> 0 1 2 1 9.16615 + #> 1 0 2 1 6.43408 + #> 2 1 2 1 7.94387 + #> 3 1 2 1 15.92867 + #> 4 0 1 1 7.13692 + #> … … … … … + #> 3995 0 2 0 0.00000 + #> 3996 0 2 0 0.00000 + #> 3997 0 3 0 0.00000 + #> 3998 0 1 0 0.00000 + #> 3999 0 5 2 17.16246 ``` """ # noqa: E501 return _make_data( @@ -190,7 +243,7 @@ def make_users_data( avg_sessions=avg_sessions, avg_orders_per_session=avg_orders_per_session, avg_revenue_per_order=avg_revenue_per_order, - result_type=result_type, + return_type=return_type, explode_sessions=False, ) @@ -208,7 +261,7 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["arrow"] = "arrow", + return_type: Literal["arrow"] = "arrow", ) -> pa.Table: ... @@ -225,7 +278,7 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["pandas"] = "pandas", + return_type: Literal["pandas"] = "pandas", ) -> PandasDataFrame: ... @@ -242,7 +295,7 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["polars"] = "polars", + return_type: Literal["polars"] = "polars", ) -> PolarsDataFrame: ... @@ -258,7 +311,7 @@ def make_sessions_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["arrow", "pandas", "polars"] = "arrow", + return_type: Literal["arrow", "pandas", "polars"] = "arrow", ) -> pa.Table | PandasDataFrame | PolarsDataFrame: """Generate simulated user data for A/B testing scenarios. @@ -293,7 +346,7 @@ def make_sessions_data( avg_orders_per_session: Average number of orders per session. Should be less than `1`. avg_revenue_per_order: Average revenue per order. - result_type: Result type. + return_type: Result type. Result types: - `"arrow"`: PyArrow Table. @@ -310,6 +363,50 @@ def make_sessions_data( data = tt.make_sessions_data(seed=42) data + #> pyarrow.Table + #> user: int64 + #> variant: int64 + #> sessions: int64 + #> orders: int64 + #> revenue: double + #> ---- + #> user: [[0,0,1,1,2,...,3999,3999,3999,3999,3999]] + #> variant: [[1,1,0,0,1,...,0,0,0,0,0]] + #> sessions: [[1,1,1,1,1,...,1,1,1,1,1]] + #> orders: [[1,1,1,1,1,...,1,0,1,1,0]] + #> revenue: [[5.88717816119309,6.131079903793326,2.614675492093661,12.296074812201192,11.573409274639534,...,23.63494099585371,0,2.396078290493153,24.538111422839766,0]] + ``` + + With covariates: + + ```python + data = tt.make_sessions_data(seed=42, covariates=True) + data + #> pyarrow.Table + #> user: int64 + #> variant: int64 + #> sessions: int64 + #> orders: int64 + #> revenue: double + #> sessions_covariate: double + #> orders_covariate: double + #> revenue_covariate: double + #> ---- + #> user: [[0,0,1,1,2,...,3999,3999,3999,3999,3999]] + #> variant: [[1,1,0,0,1,...,0,0,0,0,0]] + #> sessions: [[1,1,1,1,1,...,1,1,1,1,1]] + #> orders: [[1,1,1,1,1,...,1,0,1,1,0]] + #> revenue: [[5.88717816119309,6.131079903793326,2.614675492093661,12.296074812201192,11.573409274639534,...,23.63494099585371,0,2.396078290493153,24.538111422839766,0]] + #> sessions_covariate: [[1.5,1.5,0,0,1.5,...,0.2,0.2,0.2,0.2,0.2]] + #> orders_covariate: [[0.5,0.5,0,0,1.5,...,0,0,0,0,0]] + #> revenue_covariate: [[1.2367323749905585,1.2367323749905585,0,0,12.324434081065741,...,0,0,0,0,0]] + ``` + + As Pandas DataFrame: + + ```python + data = tt.make_sessions_data(seed=42, return_type="pandas") + print(data) #> user variant sessions orders revenue #> 0 0 1 1 1 5.887178 #> 1 0 1 1 1 6.131080 @@ -326,25 +423,34 @@ def make_sessions_data( #> [7958 rows x 5 columns] ``` - With covariates: + As Polars DataFrame: ```python - data = tt.make_sessions_data(seed=42, covariates=True) - data - #> user variant sessions orders revenue sessions_covariate orders_covariate revenue_covariate - #> 0 0 1 1 1 5.887178 1.5 0.5 1.236732 - #> 1 0 1 1 1 6.131080 1.5 0.5 1.236732 - #> 2 1 0 1 1 2.614675 0.0 0.0 0.000000 - #> 3 1 0 1 1 12.296075 0.0 0.0 0.000000 - #> 4 2 1 1 1 11.573409 1.5 1.5 12.324434 - #> ... ... ... ... ... ... ... ... ... - #> 7953 3999 0 1 1 23.634941 0.2 0.0 0.000000 - #> 7954 3999 0 1 0 0.000000 0.2 0.0 0.000000 - #> 7955 3999 0 1 1 2.396078 0.2 0.0 0.000000 - #> 7956 3999 0 1 1 24.538111 0.2 0.0 0.000000 - #> 7957 3999 0 1 0 0.000000 0.2 0.0 0.000000 - #> - #> [7958 rows x 8 columns] + import polars as pl + + data = tt.make_sessions_data(seed=42, return_type="polars") + with pl.Config( + float_precision=5, + tbl_cell_alignment="RIGHT", + tbl_formatting="NOTHING", + trim_decimal_zeros=False, + ): + print(data) + #> shape: (7_958, 5) + #> user variant sessions orders revenue + #> --- --- --- --- --- + #> i64 i64 i64 i64 f64 + #> 0 1 1 1 5.88718 + #> 0 1 1 1 6.13108 + #> 1 0 1 1 2.61468 + #> 1 0 1 1 12.29607 + #> 2 1 1 1 11.57341 + #> … … … … … + #> 3999 0 1 1 23.63494 + #> 3999 0 1 0 0.00000 + #> 3999 0 1 1 2.39608 + #> 3999 0 1 1 24.53811 + #> 3999 0 1 0 0.00000 ``` """ # noqa: E501 return _make_data( @@ -358,7 +464,7 @@ def make_sessions_data( avg_sessions=avg_sessions, avg_orders_per_session=avg_orders_per_session, avg_revenue_per_order=avg_revenue_per_order, - result_type=result_type, + return_type=return_type, explode_sessions=True, ) @@ -375,7 +481,7 @@ def _make_data( avg_sessions: float | int = 2, avg_orders_per_session: float = 0.25, avg_revenue_per_order: float | int = 10, - result_type: Literal["arrow", "pandas", "polars"] = "arrow", + return_type: Literal["arrow", "pandas", "polars"] = "arrow", explode_sessions: bool = False, ) -> pa.Table | PandasDataFrame | PolarsDataFrame: _check_params( @@ -468,10 +574,10 @@ def _make_data( "revenue_covariate": revenue_covariate, } - if result_type == "pandas": + if return_type == "pandas": import pandas as pd return pd.DataFrame(data) - if result_type == "polars": + if return_type == "polars": import polars as pl return pl.DataFrame(data) return pa.table(data) diff --git a/src/tea_tasting/experiment.py b/src/tea_tasting/experiment.py index 07d9445..8249a11 100644 --- a/src/tea_tasting/experiment.py +++ b/src/tea_tasting/experiment.py @@ -157,7 +157,7 @@ def __init__( #> revenue_per_user 5.24 5.73 9.3% [-2.4%, 22%] 0.123 ``` - Using the first argument `metrics` which accepts metrics if a form of dictionary: + Using the first argument `metrics` which accepts metrics in a form of dictionary: ```python experiment = tt.Experiment({ diff --git a/src/tea_tasting/metrics/proportion.py b/src/tea_tasting/metrics/proportion.py index 00634e7..2633733 100644 --- a/src/tea_tasting/metrics/proportion.py +++ b/src/tea_tasting/metrics/proportion.py @@ -71,7 +71,7 @@ def __init__( data = tt.make_users_data(seed=42) result = experiment.analyze(data) - print(result.to_string(("control", "treatment", "pvalue"))) + print(result.to_string(("metric", "control", "treatment", "pvalue"))) #> metric control treatment pvalue #> sample_ratio 2023 1977 0.477 ``` @@ -88,7 +88,7 @@ def __init__( data = tt.make_users_data(seed=42) result = experiment.analyze(data) - print(result.to_string(("control", "treatment", "pvalue"))) + print(result.to_string(("metric", "control", "treatment", "pvalue"))) #> metric control treatment pvalue #> sample_ratio 2023 1977 3.26e-103 ``` diff --git a/src/tea_tasting/multiplicity.py b/src/tea_tasting/multiplicity.py index f691b16..fad35b4 100644 --- a/src/tea_tasting/multiplicity.py +++ b/src/tea_tasting/multiplicity.py @@ -102,8 +102,18 @@ def adjust_fdr( data = pd.concat(( - tt.make_users_data(seed=42, orders_uplift=0.10, revenue_uplift=0.15), - tt.make_users_data(seed=21, orders_uplift=0.15, revenue_uplift=0.20) + tt.make_users_data( + seed=42, + orders_uplift=0.10, + revenue_uplift=0.15, + return_type="pandas", + ), + tt.make_users_data( + seed=21, + orders_uplift=0.15, + revenue_uplift=0.20, + return_type="pandas", + ) .query("variant==1") .assign(variant=2), )) @@ -265,8 +275,18 @@ def adjust_fwer( data = pd.concat(( - tt.make_users_data(seed=42, orders_uplift=0.10, revenue_uplift=0.15), - tt.make_users_data(seed=21, orders_uplift=0.15, revenue_uplift=0.20) + tt.make_users_data( + seed=42, + orders_uplift=0.10, + revenue_uplift=0.15, + return_type="pandas", + ), + tt.make_users_data( + seed=21, + orders_uplift=0.15, + revenue_uplift=0.20, + return_type="pandas", + ) .query("variant==1") .assign(variant=2), )) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index f09b73f..8d6cb06 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -29,7 +29,7 @@ def test_make_users_data_default(): def test_make_users_data_pandas(): n_users = 100 data = tea_tasting.datasets.make_users_data( - seed=42, n_users=n_users, result_type="pandas") + seed=42, n_users=n_users, return_type="pandas") assert isinstance(data, pd.DataFrame) assert data.columns.to_list() == [ "user", "variant", "sessions", "orders", "revenue"] @@ -38,7 +38,7 @@ def test_make_users_data_pandas(): def test_make_users_data_polars(): n_users = 100 data = tea_tasting.datasets.make_users_data( - seed=42, n_users=n_users, result_type="polars") + seed=42, n_users=n_users, return_type="polars") assert isinstance(data, pl.DataFrame) assert data.columns == [ "user", "variant", "sessions", "orders", "revenue"] @@ -88,7 +88,7 @@ def test_make_sessions_data_default(): def test_make_sessions_data_pandas(): n_users = 100 data = tea_tasting.datasets.make_sessions_data( - seed=42, n_users=n_users, result_type="pandas") + seed=42, n_users=n_users, return_type="pandas") assert isinstance(data, pd.DataFrame) assert data.columns.to_list() == [ "user", "variant", "sessions", "orders", "revenue"] @@ -97,7 +97,7 @@ def test_make_sessions_data_pandas(): def test_make_sessions_data_polars(): n_users = 100 data = tea_tasting.datasets.make_sessions_data( - seed=42, n_users=n_users, result_type="polars") + seed=42, n_users=n_users, return_type="polars") assert isinstance(data, pl.DataFrame) assert data.columns == [ "user", "variant", "sessions", "orders", "revenue"]
ab