From c0153adf2a6f4a8528e76f74a640b3afa119f90c Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Thu, 8 Aug 2024 19:22:15 -0400 Subject: [PATCH 1/8] feat(main): ability to specify concurrency in run_experiment and evaluate_experiment (Arize-AI#4186) --- src/phoenix/experiments/functions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index ab95a8cc7f..65744753a7 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -108,6 +108,7 @@ def run_experiment( rate_limit_errors: Optional[RateLimitErrors] = None, dry_run: Union[bool, int] = False, print_summary: bool = True, + concurrency: int = 3 ) -> RanExperiment: """ Runs an experiment using a given set of dataset of examples. @@ -158,6 +159,8 @@ def run_experiment( examples of the given size. Defaults to False. print_summary (bool): Whether to print a summary of the experiment and evaluation results. Defaults to True. + concurrency (int): concurrency for task execution. + Defaults to 3 Returns: RanExperiment: The results of the experiment and evaluation. Additional evaluations can be @@ -389,6 +392,7 @@ async def async_run_experiment(test_case: TestCase) -> ExperimentRun: exit_on_error=False, fallback_return_value=None, tqdm_bar_format=get_tqdm_progress_bar_formatter("running tasks"), + concurrency=concurrency ) test_cases = [ @@ -426,7 +430,8 @@ def evaluate_experiment( *, dry_run: Union[bool, int] = False, print_summary: bool = True, - rate_limit_errors: Optional[RateLimitErrors] = None, + rate_limit_errors: Optional[RateLimitErrors] = None,, + concurrency: int = 3 ) -> RanExperiment: if not dry_run and _is_dry_run(experiment): dry_run = True @@ -628,6 +633,7 @@ async def async_evaluate_run( exit_on_error=False, fallback_return_value=None, tqdm_bar_format=get_tqdm_progress_bar_formatter("running experiment evaluations"), + concurrency=concurrency ) eval_runs, _execution_details = executor.run(evaluation_input) eval_summary = EvaluationSummary.from_eval_runs( From 4240c68734837beb9e1945df0d20e3c469f2f980 Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Mon, 12 Aug 2024 04:39:47 -0400 Subject: [PATCH 2/8] fix(main): improve concurrency docs & fix typo --- src/phoenix/experiments/functions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index 65744753a7..8fc45db61b 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -159,7 +159,8 @@ def run_experiment( examples of the given size. Defaults to False. print_summary (bool): Whether to print a summary of the experiment and evaluation results. Defaults to True. - concurrency (int): concurrency for task execution. + concurrency (int): concurrency for task execution. The task must be an async routine for + concurrency support. Defaults to 3 Returns: @@ -430,7 +431,7 @@ def evaluate_experiment( *, dry_run: Union[bool, int] = False, print_summary: bool = True, - rate_limit_errors: Optional[RateLimitErrors] = None,, + rate_limit_errors: Optional[RateLimitErrors] = None, concurrency: int = 3 ) -> RanExperiment: if not dry_run and _is_dry_run(experiment): From 233f8281f4cfb5b1ea5a0722ddf46628bfa708f5 Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Mon, 12 Aug 2024 04:41:31 -0400 Subject: [PATCH 3/8] fix typo --- src/phoenix/experiments/functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index 8fc45db61b..113f8a0f31 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -159,8 +159,8 @@ def run_experiment( examples of the given size. Defaults to False. print_summary (bool): Whether to print a summary of the experiment and evaluation results. Defaults to True. - concurrency (int): concurrency for task execution. The task must be an async routine for - concurrency support. + concurrency (int): concurrency for task execution. The task must be an async coroutine + for concurrency support. Defaults to 3 Returns: From 62f7d5829d00fa0833e7a2ad60275aba869bf53e Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Mon, 12 Aug 2024 12:41:13 -0400 Subject: [PATCH 4/8] clarified run_experiment concurrency docstring --- src/phoenix/experiments/functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index 113f8a0f31..67424ea51f 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -159,8 +159,8 @@ def run_experiment( examples of the given size. Defaults to False. print_summary (bool): Whether to print a summary of the experiment and evaluation results. Defaults to True. - concurrency (int): concurrency for task execution. The task must be an async coroutine - for concurrency support. + concurrency (int): Specifies the concurrency for task execution. In order to enable + concurrent task execution, the task callable must be a coroutine function. Defaults to 3 Returns: From 7984fd7182c3858c391dde89ba4e38c932f9ed95 Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Mon, 12 Aug 2024 12:43:01 -0400 Subject: [PATCH 5/8] fix typo --- src/phoenix/experiments/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index 67424ea51f..a743b42e3f 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -161,7 +161,7 @@ def run_experiment( Defaults to True. concurrency (int): Specifies the concurrency for task execution. In order to enable concurrent task execution, the task callable must be a coroutine function. - Defaults to 3 + Defaults to 3. Returns: RanExperiment: The results of the experiment and evaluation. Additional evaluations can be From aa1997a1317efcb22e579e413d53efccda309883 Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Tue, 13 Aug 2024 05:06:14 -0400 Subject: [PATCH 6/8] pass concurrency to evaluate_experiment --- src/phoenix/experiments/functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index a743b42e3f..dbb9f7afe0 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -393,7 +393,7 @@ async def async_run_experiment(test_case: TestCase) -> ExperimentRun: exit_on_error=False, fallback_return_value=None, tqdm_bar_format=get_tqdm_progress_bar_formatter("running tasks"), - concurrency=concurrency + concurrency=concurrency, ) test_cases = [ @@ -419,6 +419,7 @@ async def async_run_experiment(test_case: TestCase) -> ExperimentRun: dry_run=dry_run, print_summary=print_summary, rate_limit_errors=rate_limit_errors, + concurrency=concurrency, ) if print_summary: print(ran_experiment) From da6bf683d31eea83e471d7e94d8dd2c4eb3a09f1 Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Tue, 13 Aug 2024 11:05:34 -0400 Subject: [PATCH 7/8] lint fix --- src/phoenix/experiments/functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index dbb9f7afe0..19b3ea43dc 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -108,7 +108,7 @@ def run_experiment( rate_limit_errors: Optional[RateLimitErrors] = None, dry_run: Union[bool, int] = False, print_summary: bool = True, - concurrency: int = 3 + concurrency: int = 3, ) -> RanExperiment: """ Runs an experiment using a given set of dataset of examples. @@ -433,7 +433,7 @@ def evaluate_experiment( dry_run: Union[bool, int] = False, print_summary: bool = True, rate_limit_errors: Optional[RateLimitErrors] = None, - concurrency: int = 3 + concurrency: int = 3, ) -> RanExperiment: if not dry_run and _is_dry_run(experiment): dry_run = True @@ -635,7 +635,7 @@ async def async_evaluate_run( exit_on_error=False, fallback_return_value=None, tqdm_bar_format=get_tqdm_progress_bar_formatter("running experiment evaluations"), - concurrency=concurrency + concurrency=concurrency, ) eval_runs, _execution_details = executor.run(evaluation_input) eval_summary = EvaluationSummary.from_eval_runs( From db7c7559720dc5614408f98ba16636ce4be9bb6f Mon Sep 17 00:00:00 2001 From: Anton Abilov Date: Tue, 13 Aug 2024 12:17:54 -0400 Subject: [PATCH 8/8] fix line ending lint --- src/phoenix/experiments/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/phoenix/experiments/functions.py b/src/phoenix/experiments/functions.py index 19b3ea43dc..20c1adb381 100644 --- a/src/phoenix/experiments/functions.py +++ b/src/phoenix/experiments/functions.py @@ -159,7 +159,7 @@ def run_experiment( examples of the given size. Defaults to False. print_summary (bool): Whether to print a summary of the experiment and evaluation results. Defaults to True. - concurrency (int): Specifies the concurrency for task execution. In order to enable + concurrency (int): Specifies the concurrency for task execution. In order to enable concurrent task execution, the task callable must be a coroutine function. Defaults to 3.