Skip to content

Commit 84f467f

Browse files
omatthew98Future-Outlier
authored andcommitted
[data] Use ranges in test_operator_fusion.py (ray-project#58000)
## Description We are using `read_parquet` in two of our tests in `test_operator_fusion.py`, this switches those to use `range` to make the tests less brittle. Signed-off-by: Matthew Owen <mowen@anyscale.com> Signed-off-by: Future-Outlier <eric901201@gmail.com>
1 parent 58cfe4d commit 84f467f

File tree

1 file changed

+7
-5
lines changed

1 file changed

+7
-5
lines changed

python/ray/data/tests/test_operator_fusion.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,8 @@ def test_read_with_map_batches_fused_successfully(
259259

260260
# Test that fusion of map operators merges their block sizes in the expected way
261261
# (taking the max).
262-
ds = ray.data.read_parquet(temp_dir)
262+
n = 10
263+
ds = ray.data.range(n)
263264

264265
mapped_ds = ds.map_batches(lambda x: x).map_batches(lambda x: x)
265266

@@ -273,7 +274,7 @@ def test_read_with_map_batches_fused_successfully(
273274
# All Map ops are fused with Read
274275
assert (
275276
"InputDataBuffer[Input] -> "
276-
"TaskPoolMapOperator[ReadParquet->MapBatches(<lambda>)->MapBatches(<lambda>)]"
277+
"TaskPoolMapOperator[ReadRange->MapBatches(<lambda>)->MapBatches(<lambda>)]"
277278
== actual_plan_str
278279
)
279280

@@ -376,7 +377,8 @@ def test_map_batches_with_batch_size_specified_fusion(
376377
):
377378
# Test that fusion of map operators merges their block sizes in the expected way
378379
# (taking the max).
379-
ds = ray.data.read_parquet(temp_dir)
380+
n = 10
381+
ds = ray.data.range(n)
380382

381383
mapped_ds = ds.map_batches(
382384
lambda x: x,
@@ -397,14 +399,14 @@ def test_map_batches_with_batch_size_specified_fusion(
397399
expected_min_rows_per_bundle = None
398400
expected_plan_str = (
399401
"InputDataBuffer[Input] -> "
400-
"TaskPoolMapOperator[ReadParquet->MapBatches(<lambda>)->MapBatches(<lambda>)]"
402+
"TaskPoolMapOperator[ReadRange->MapBatches(<lambda>)->MapBatches(<lambda>)]"
401403
)
402404
else:
403405
expected_min_rows_per_bundle = max(
404406
upstream_batch_size or 0, downstream_batch_size or 0
405407
)
406408
expected_plan_str = (
407-
"InputDataBuffer[Input] -> TaskPoolMapOperator[ReadParquet] -> "
409+
"InputDataBuffer[Input] -> TaskPoolMapOperator[ReadRange] -> "
408410
"TaskPoolMapOperator[MapBatches(<lambda>)->MapBatches(<lambda>)]"
409411
)
410412

0 commit comments

Comments
 (0)