Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(experiments): ability to specify concurrency in run_experiment and evaluate_experiment #4189

Merged
merged 8 commits into from
Aug 15, 2024
Merged

feat(experiments): ability to specify concurrency in run_experiment and evaluate_experiment #4189

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c0153ad
feat(main): ability to specify concurrency in run_experiment and eval…
anton164 Aug 8, 2024
4240c68
fix(main): improve concurrency docs & fix typo
anton164 Aug 12, 2024
233f828
fix typo
anton164 Aug 12, 2024
62f7d58
clarified run_experiment concurrency docstring
anton164 Aug 12, 2024
7984fd7
fix typo
anton164 Aug 12, 2024
aa1997a
pass concurrency to evaluate_experiment
anton164 Aug 13, 2024
da6bf68
lint fix
anton164 Aug 13, 2024
db7c755
fix line ending lint
anton164 Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/phoenix/experiments/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ def run_experiment(
rate_limit_errors: Optional[RateLimitErrors] = None,
dry_run: Union[bool, int] = False,
print_summary: bool = True,
concurrency: int = 3,
) -> RanExperiment:
"""
Runs an experiment using a given set of dataset of examples.
Expand Down Expand Up @@ -158,6 +159,9 @@ def run_experiment(
examples of the given size. Defaults to False.
print_summary (bool): Whether to print a summary of the experiment and evaluation results.
Defaults to True.
concurrency (int): Specifies the concurrency for task execution. In order to enable
concurrent task execution, the task callable must be a coroutine function.
Defaults to 3.

Returns:
RanExperiment: The results of the experiment and evaluation. Additional evaluations can be
Expand Down Expand Up @@ -389,6 +393,7 @@ async def async_run_experiment(test_case: TestCase) -> ExperimentRun:
exit_on_error=False,
fallback_return_value=None,
tqdm_bar_format=get_tqdm_progress_bar_formatter("running tasks"),
concurrency=concurrency,
)

test_cases = [
Expand All @@ -414,6 +419,7 @@ async def async_run_experiment(test_case: TestCase) -> ExperimentRun:
dry_run=dry_run,
print_summary=print_summary,
rate_limit_errors=rate_limit_errors,
concurrency=concurrency,
)
if print_summary:
print(ran_experiment)
Expand All @@ -427,6 +433,7 @@ def evaluate_experiment(
dry_run: Union[bool, int] = False,
print_summary: bool = True,
rate_limit_errors: Optional[RateLimitErrors] = None,
concurrency: int = 3,
) -> RanExperiment:
if not dry_run and _is_dry_run(experiment):
dry_run = True
Expand Down Expand Up @@ -628,6 +635,7 @@ async def async_evaluate_run(
exit_on_error=False,
fallback_return_value=None,
tqdm_bar_format=get_tqdm_progress_bar_formatter("running experiment evaluations"),
concurrency=concurrency,
)
eval_runs, _execution_details = executor.run(evaluation_input)
eval_summary = EvaluationSummary.from_eval_runs(
Expand Down
Loading