From e06a8a710b4519fcc3a01e361c4edb6c94c5e741 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sat, 27 May 2023 14:09:26 -0700 Subject: [PATCH 1/5] Mark all eager variants of async_tree benchmark as require Python>=3.12 --- .../benchmarks/bm_async_tree/bm_async_tree_eager.toml | 4 ++++ .../bm_async_tree/bm_async_tree_eager_cpu_io_mixed.toml | 4 ++++ .../benchmarks/bm_async_tree/bm_async_tree_eager_io.toml | 4 ++++ .../bm_async_tree/bm_async_tree_eager_memoization.toml | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager.toml index 09d16ee8..e4d5fd2f 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager.toml +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager.toml @@ -1,3 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + [tool.pyperformance] name = "async_tree_eager" extra_opts = ["eager"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed.toml index 4766cb23..22e36fba 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed.toml +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed.toml @@ -1,3 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + [tool.pyperformance] name = "async_tree_eager_cpu_io_mixed" extra_opts = ["eager_cpu_io_mixed"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io.toml index de1dfb2a..57bf2da1 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io.toml +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io.toml @@ -1,3 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + [tool.pyperformance] name = "async_tree_eager_io" extra_opts = ["eager_io"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization.toml index ec199382..4a8c6e63 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization.toml +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization.toml @@ -1,3 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + [tool.pyperformance] name = "async_tree_eager_memoization" extra_opts = ["eager_memoization"] From 734b2bef364eb4a559730a6f0022c9378a4da0f4 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sat, 27 May 2023 14:29:12 -0700 Subject: [PATCH 2/5] Add "asyncio" tag to all async_tree benchmarks --- pyperformance/data-files/benchmarks/bm_async_tree/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/pyproject.toml b/pyperformance/data-files/benchmarks/bm_async_tree/pyproject.toml index c1f73aac..565512a0 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/pyproject.toml +++ b/pyperformance/data-files/benchmarks/bm_async_tree/pyproject.toml @@ -7,4 +7,5 @@ dynamic = ["version"] [tool.pyperformance] name = "async_tree" +tags = "asyncio" extra_opts = ["none"] From 3ab4a544ea723e0b2cae063e613faf75e1f73c61 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sat, 27 May 2023 14:31:43 -0700 Subject: [PATCH 3/5] Add TaskGroups variants to all async_tree benchmarks --- pyperformance/data-files/benchmarks/MANIFEST | 9 +++++ .../bm_async_tree_cpu_io_mixed_tg.toml | 7 ++++ .../bm_async_tree_eager_cpu_io_mixed_tg.toml | 7 ++++ .../bm_async_tree_eager_io_tg.toml | 7 ++++ .../bm_async_tree_eager_memoization_tg.toml | 7 ++++ .../bm_async_tree/bm_async_tree_eager_tg.toml | 7 ++++ .../bm_async_tree/bm_async_tree_io_tg.toml | 7 ++++ .../bm_async_tree_memoization_tg.toml | 7 ++++ .../bm_async_tree/bm_async_tree_tg.toml | 7 ++++ .../benchmarks/bm_async_tree/run_benchmark.py | 37 +++++++++++++++---- 10 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_cpu_io_mixed_tg.toml create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed_tg.toml create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io_tg.toml create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization_tg.toml create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_tg.toml create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_io_tg.toml create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_memoization_tg.toml create mode 100644 pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_tg.toml diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST index 0a338e44..4420f98b 100644 --- a/pyperformance/data-files/benchmarks/MANIFEST +++ b/pyperformance/data-files/benchmarks/MANIFEST @@ -11,6 +11,14 @@ async_tree_eager async_tree_eager_cpu_io_mixed async_tree_eager_io async_tree_eager_memoization +async_tree_tg +async_tree_cpu_io_mixed_tg +async_tree_io_tg +async_tree_memoization_tg +async_tree_eager_tg +async_tree_eager_cpu_io_mixed_tg +async_tree_eager_io_tg +async_tree_eager_memoization_tg asyncio_tcp asyncio_tcp_ssl concurrent_imap @@ -82,6 +90,7 @@ xml_etree #[groups] +#asyncio #startup #regex #serialize diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_cpu_io_mixed_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_cpu_io_mixed_tg.toml new file mode 100644 index 00000000..c1fb778a --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_cpu_io_mixed_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.11" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_cpu_io_mixed_tg" +extra_opts = ["cpu_io_mixed", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed_tg.toml new file mode 100644 index 00000000..5d1bfeed --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_cpu_io_mixed_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_eager_cpu_io_mixed_tg" +extra_opts = ["eager_cpu_io_mixed", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io_tg.toml new file mode 100644 index 00000000..e586e87d --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_io_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_eager_io_tg" +extra_opts = ["eager_io", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization_tg.toml new file mode 100644 index 00000000..d24a3fa6 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_memoization_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_eager_memoization_tg" +extra_opts = ["eager_memoization", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_tg.toml new file mode 100644 index 00000000..868414c9 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_eager_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.12" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_eager_tg" +extra_opts = ["eager", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_io_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_io_tg.toml new file mode 100644 index 00000000..7a3e22d9 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_io_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.11" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_io_tg" +extra_opts = ["io", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_memoization_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_memoization_tg.toml new file mode 100644 index 00000000..7963305d --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_memoization_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.11" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_memoization_tg" +extra_opts = ["memoization", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_tg.toml b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_tg.toml new file mode 100644 index 00000000..0c7495be --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_async_tree/bm_async_tree_tg.toml @@ -0,0 +1,7 @@ +[project] +requires-python = ">=3.11" +dynamic = ["version"] + +[tool.pyperformance] +name = "async_tree_tg" +extra_opts = ["none", "--task-groups"] diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py index 72fc917c..d0407140 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py +++ b/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py @@ -12,8 +12,8 @@ the other half simulate the same workload as the "memoization" variant. -All variants also have an "eager" flavor that uses -the asyncio eager task factory (if available). +All variants also have an "eager" flavor that uses the asyncio eager task +factory (if available), and a "tg" variant that uses TaskGroups. """ @@ -34,8 +34,9 @@ class AsyncTree: - def __init__(self): + def __init__(self, use_task_groups=False): self.cache = {} + self.use_task_groups = use_task_groups # set to deterministic random, so that the results are reproducible random.seed(RANDOM_SEED) @@ -47,17 +48,31 @@ async def workload_func(self): "To be implemented by each variant's derived class." ) - async def recurse(self, recurse_level): + async def recurse_with_gather(self, recurse_level): if recurse_level == 0: await self.workload_func() return await asyncio.gather( - *[self.recurse(recurse_level - 1) for _ in range(NUM_RECURSE_BRANCHES)] + *[self.recurse_with_gather(recurse_level - 1) + for _ in range(NUM_RECURSE_BRANCHES)] + ) + + async def recurse_with_task_groups(self, recurse_level): + if recurse_level == 0: + await self.workload_func() + return + + await asyncio.gather( + *[self.recurse_with_task_groups(recurse_level - 1) + for _ in range(NUM_RECURSE_BRANCHES)] ) async def run(self): - await self.recurse(NUM_RECURSE_LEVELS) + if self.use_task_groups: + await self.recurse_with_task_groups(NUM_RECURSE_LEVELS) + else: + await self.recurse_with_gather(NUM_RECURSE_LEVELS) class EagerMixin: @@ -132,6 +147,8 @@ def add_metadata(runner): def add_cmdline_args(cmd, args): cmd.append(args.benchmark) + if args.task_groups: + cmd.append("--task-groups") def add_parser_args(parser): @@ -149,6 +166,12 @@ def add_parser_args(parser): "memoization" variant. """, ) + parser.add_argument( + "--task-groups", + action="store_true", + default=False, + help="Use TaskGroups instead of gather.", + ) BENCHMARKS = { @@ -171,5 +194,5 @@ def add_parser_args(parser): benchmark = args.benchmark async_tree_class = BENCHMARKS[benchmark] - async_tree = async_tree_class() + async_tree = async_tree_class(use_task_groups=args.task_groups) runner.bench_async_func(f"async_tree_{benchmark}", async_tree.run) From d93e0ffc865d27b0f66ae826c46bfc57422299b6 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sun, 28 May 2023 00:16:26 -0700 Subject: [PATCH 4/5] Actually use TaskGroup in the tg variant --- .../benchmarks/bm_async_tree/run_benchmark.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py index d0407140..7e7a7183 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py +++ b/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py @@ -58,19 +58,19 @@ async def recurse_with_gather(self, recurse_level): for _ in range(NUM_RECURSE_BRANCHES)] ) - async def recurse_with_task_groups(self, recurse_level): + async def recurse_with_task_group(self, recurse_level): if recurse_level == 0: await self.workload_func() return - await asyncio.gather( - *[self.recurse_with_task_groups(recurse_level - 1) - for _ in range(NUM_RECURSE_BRANCHES)] - ) + async with asyncio.TaskGroup() as tg: + for _ in range(NUM_RECURSE_BRANCHES): + tg.create_task( + self.recurse_with_task_group(recurse_level - 1)) async def run(self): if self.use_task_groups: - await self.recurse_with_task_groups(NUM_RECURSE_LEVELS) + await self.recurse_with_task_group(NUM_RECURSE_LEVELS) else: await self.recurse_with_gather(NUM_RECURSE_LEVELS) From 1d6bf0625c76e6520d4638036b635f43bbf6e4dd Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sun, 28 May 2023 00:48:04 -0700 Subject: [PATCH 5/5] Append "_tg" to benchmark name in runner.bench_async_func call --- .../data-files/benchmarks/bm_async_tree/run_benchmark.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py index 7e7a7183..131e59c1 100644 --- a/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py +++ b/pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py @@ -195,4 +195,7 @@ def add_parser_args(parser): async_tree_class = BENCHMARKS[benchmark] async_tree = async_tree_class(use_task_groups=args.task_groups) - runner.bench_async_func(f"async_tree_{benchmark}", async_tree.run) + bench_name = f"async_tree_{benchmark}" + if args.task_groups: + bench_name += "_tg" + runner.bench_async_func(bench_name, async_tree.run)