From 69eea79aa9407552e6408e525b3e0561d3e4222e Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 09:46:47 +0100 Subject: [PATCH 1/8] [ArrayManager] TST: include subset of ArrayManager tests in all CI builds --- ci/run_tests.sh | 5 +++++ pandas/conftest.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index ec4c87e8c91b0..169b30c5529fc 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -29,3 +29,8 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" + +PYTEST_AM_CMD = "PANDAS_DATA_MANAGER=array ${XVFB}pytest -m -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" + +echo $PYTEST_AM_CMD +sh -c "$PYTEST_AM_CMD" diff --git a/pandas/conftest.py b/pandas/conftest.py index 688ad6dcc5e48..f3356d2998ff8 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -85,6 +85,9 @@ def pytest_configure(config): config.addinivalue_line( "markers", "arm_slow: mark a test as slow for arm64 architecture" ) + config.addinivalue_line( + "markers", "arraymanager: mark a test to run with ArrayManager enabled" + ) def pytest_addoption(parser): @@ -121,6 +124,13 @@ def pytest_runtest_setup(item): pytest.skip("skipping high memory test since --run-high-memory was not set") +def pytest_collection_modifyitems(items): + for item in items: + # mark all tests in the pandas/tests/frame directory with "arraymanager" + if "/frame/" in item.nodeid: + item.add_marker(pytest.mark.arraymanager) + + # Hypothesis hypothesis.settings.register_profile( "ci", From c63c5eea80292d08090683cbf1a43fda0d4c211f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 09:52:24 +0100 Subject: [PATCH 2/8] fixup command --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 169b30c5529fc..9620dae61ce39 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -30,7 +30,7 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" -PYTEST_AM_CMD = "PANDAS_DATA_MANAGER=array ${XVFB}pytest -m -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" +PYTEST_AM_CMD = "PANDAS_DATA_MANAGER=array ${XVFB}pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" echo $PYTEST_AM_CMD sh -c "$PYTEST_AM_CMD" From 6b7820f082aeba123788dc0f8b12a50708c32741 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 18 Mar 2021 10:34:11 +0100 Subject: [PATCH 3/8] remove spaces --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 9620dae61ce39..635ef9bc19d97 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -30,7 +30,7 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" -PYTEST_AM_CMD = "PANDAS_DATA_MANAGER=array ${XVFB}pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" +PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array ${XVFB}pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" echo $PYTEST_AM_CMD sh -c "$PYTEST_AM_CMD" From 5388c703563ebe9d0deb1c0bfce9abd62dbdd2b6 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 19 Mar 2021 09:56:24 +0100 Subject: [PATCH 4/8] remove second XVFB --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 635ef9bc19d97..d73940c1010ad 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -30,7 +30,7 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" -PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array ${XVFB}pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" +PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" echo $PYTEST_AM_CMD sh -c "$PYTEST_AM_CMD" From 00c0554eaf30d74de09294ab4861bba1e4a30402 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 19 Mar 2021 10:01:40 +0100 Subject: [PATCH 5/8] ignore mean of empty slice warning --- pandas/tests/frame/test_reductions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index d24320ad17709..fb0e28b7f432a 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -382,7 +382,8 @@ def kurt(x): pass # TODO: Ensure warning isn't emitted in the first place - @pytest.mark.filterwarnings("ignore:All-NaN:RuntimeWarning") + # ignore mean of empty slice and all-NaN + # @pytest.mark.filterwarnings("ignore:RuntimeWarning") def test_median(self, float_frame_with_na, int_frame): def wrapper(x): if isna(x).any(): From d0a65c23669ee2ab9062efb3febcda09fecc9f33 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 19 Mar 2021 10:06:41 +0100 Subject: [PATCH 6/8] 32bit fixes --- pandas/tests/frame/indexing/test_setitem.py | 4 +++- pandas/tests/frame/test_stack_unstack.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/frame/indexing/test_setitem.py b/pandas/tests/frame/indexing/test_setitem.py index e8cdcfcaafa86..361312c3c2d1b 100644 --- a/pandas/tests/frame/indexing/test_setitem.py +++ b/pandas/tests/frame/indexing/test_setitem.py @@ -926,7 +926,9 @@ def test_setitem_duplicate_columns_not_inplace(self): tm.assert_frame_equal(df_view, df_copy) tm.assert_frame_equal(df, expected) - @pytest.mark.parametrize("value", [1, np.array([[1], [1]]), [[1], [1]]]) + @pytest.mark.parametrize( + "value", [1, np.array([[1], [1]], dtype="int64"), [[1], [1]]] + ) def test_setitem_same_dtype_not_inplace(self, value, using_array_manager, request): # GH#39510 if not using_array_manager: diff --git a/pandas/tests/frame/test_stack_unstack.py b/pandas/tests/frame/test_stack_unstack.py index 253f416bd6f18..cc4042822bc8b 100644 --- a/pandas/tests/frame/test_stack_unstack.py +++ b/pandas/tests/frame/test_stack_unstack.py @@ -905,7 +905,7 @@ def test_unstack_nan_index3(self, using_array_manager): if using_array_manager: # INFO(ArrayManager) with ArrayManager preserve dtype where possible cols = right.columns[[1, 2, 3, 5]] - right[cols] = right[cols].astype("int64") + right[cols] = right[cols].astype(df["C"].dtype) tm.assert_frame_equal(left, right) def test_unstack_nan_index4(self): From 81c10c329d6e6998307088aaedc12f42be86206f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 19 Mar 2021 15:29:16 +0100 Subject: [PATCH 7/8] uncomment --- pandas/tests/frame/test_reductions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index fb0e28b7f432a..3bb4748488408 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -383,7 +383,7 @@ def kurt(x): # TODO: Ensure warning isn't emitted in the first place # ignore mean of empty slice and all-NaN - # @pytest.mark.filterwarnings("ignore:RuntimeWarning") + @pytest.mark.filterwarnings("ignore:RuntimeWarning") def test_median(self, float_frame_with_na, int_frame): def wrapper(x): if isna(x).any(): From 113bb78d30fef21fcaa84f6b4afa4a88dae81804 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 23 Mar 2021 12:09:10 +0100 Subject: [PATCH 8/8] fix filterwarnings --- pandas/tests/frame/test_reductions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index 3bb4748488408..255ff1fc69eeb 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -383,7 +383,7 @@ def kurt(x): # TODO: Ensure warning isn't emitted in the first place # ignore mean of empty slice and all-NaN - @pytest.mark.filterwarnings("ignore:RuntimeWarning") + @pytest.mark.filterwarnings("ignore::RuntimeWarning") def test_median(self, float_frame_with_na, int_frame): def wrapper(x): if isna(x).any():