From cc00c24a0e2fc549206546807dc01d3fd16edaaa Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 26 Apr 2023 17:45:10 -0700 Subject: [PATCH 1/8] CI: Add testing for Musl Linux --- .github/workflows/unit-tests.yml | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 69baa3ccd6f34..5e6c9c6c41ef6 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -241,6 +241,41 @@ jobs: group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-32bit cancel-in-progress: true + Linux-Musl: + runs-on: ubuntu-22.04 + container: + image: quay.io/pypa/musllinux_1_1_x86_64 + steps: + - name: Checkout pandas Repo + # actions/checkout does not work since it requires node + run: | + git config --global --add safe.directory $PWD + + if [ $GITHUB_EVENT_NAME != pull_request ]; then + git clone --recursive --branch=$GITHUB_REF_NAME https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE + git reset --hard $GITHUB_SHA + else + git clone --recursive https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE + git fetch origin $GITHUB_REF:my_ref_name + git checkout $GITHUB_BASE_REF + git -c user.email="you@example.com" merge --no-commit my_ref_name + fi + - name: Build environment and Run Tests + run: | + /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev + . ~/virtualenvs/pandas-dev/bin/activate + python -m pip install --no-cache-dir --no-deps -U pip wheel setuptools + python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1 + python setup.py build_ext -q -j$(nproc) + python -m pip install --no-cache-dir --no-build-isolation --no-use-pep517 -e . + python -m pip list + export PANDAS_CI=1 + python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-musl + cancel-in-progress: true + python-dev: # This job may or may not run depending on the state of the next # unreleased Python version. DO NOT DELETE IT. From ce109341e163a4db9e3219979f9ae05b0c2693e3 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 27 Apr 2023 09:36:24 -0700 Subject: [PATCH 2/8] Install locale --- .github/workflows/unit-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5e6c9c6c41ef6..578e5030f36a0 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -260,6 +260,11 @@ jobs: git checkout $GITHUB_BASE_REF git -c user.email="you@example.com" merge --no-commit my_ref_name fi + - name: Configure System Packages + run: | + apk update + apk add musl-locales + apk add lang - name: Build environment and Run Tests run: | /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev From 1a6344b5b6bc963735618485d239a02d1abe89fe Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:21:04 -0700 Subject: [PATCH 3/8] Add loc path --- .github/workflows/unit-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 578e5030f36a0..08f42a27babed 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -262,9 +262,9 @@ jobs: fi - name: Configure System Packages run: | + export MUSL_LOCPATH=/usr/share/i18n/locales/musl apk update - apk add musl-locales - apk add lang + apk add musl-locales musl-locales-lang - name: Build environment and Run Tests run: | /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev From 1ed91b7d1794f68d8818bf18c2880b2fdb271515 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:30:13 -0700 Subject: [PATCH 4/8] set tz too --- .github/workflows/unit-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 08f42a27babed..5a87819e209bb 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -263,6 +263,7 @@ jobs: - name: Configure System Packages run: | export MUSL_LOCPATH=/usr/share/i18n/locales/musl + export TZ=GMT apk update apk add musl-locales musl-locales-lang - name: Build environment and Run Tests From cc882db3c169a4babd2b3a71f448f576f548d598 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 27 Apr 2023 14:57:03 -0700 Subject: [PATCH 5/8] Skip one test for musl, use more standard tz for musl --- .github/workflows/unit-tests.yml | 1 - pandas/compat/__init__.py | 2 ++ pandas/compat/_constants.py | 3 +++ pandas/tests/config/test_localization.py | 16 +++++++++++++++- pandas/tests/tslibs/test_parsing.py | 2 +- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5a87819e209bb..08f42a27babed 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -263,7 +263,6 @@ jobs: - name: Configure System Packages run: | export MUSL_LOCPATH=/usr/share/i18n/locales/musl - export TZ=GMT apk update apk add musl-locales musl-locales-lang - name: Build environment and Run Tests diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index 3d7589bf67ee2..00957c45a7fbe 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -16,6 +16,7 @@ from pandas.compat._constants import ( IS64, + ISMUSL, PY310, PY311, PYPY, @@ -160,6 +161,7 @@ def get_lzma_file() -> type[pandas.compat.compressors.LZMAFile]: "pa_version_under9p0", "pa_version_under11p0", "IS64", + "ISMUSL", "PY310", "PY311", "PYPY", diff --git a/pandas/compat/_constants.py b/pandas/compat/_constants.py index 1d522a5b4cd09..1d7fe23b3d2ea 100644 --- a/pandas/compat/_constants.py +++ b/pandas/compat/_constants.py @@ -9,16 +9,19 @@ import platform import sys +import sysconfig IS64 = sys.maxsize > 2**32 PY310 = sys.version_info >= (3, 10) PY311 = sys.version_info >= (3, 11) PYPY = platform.python_implementation() == "PyPy" +ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "") __all__ = [ "IS64", + "ISMUSL", "PY310", "PY311", "PYPY", diff --git a/pandas/tests/config/test_localization.py b/pandas/tests/config/test_localization.py index 6474177261b91..8ea3085116eaa 100644 --- a/pandas/tests/config/test_localization.py +++ b/pandas/tests/config/test_localization.py @@ -10,6 +10,8 @@ set_locale, ) +from pandas.compat import ISMUSL + import pandas as pd _all_locales = get_locales() @@ -46,7 +48,19 @@ def test_can_set_locale_valid_set(lc_var): assert before_locale == after_locale -@pytest.mark.parametrize("lc_var", (locale.LC_ALL, locale.LC_CTYPE, locale.LC_TIME)) +@pytest.mark.parametrize( + "lc_var", + ( + locale.LC_ALL, + pytest.param( + locale.LC_CTYPE, + marks=pytest.mark.skipif( + ISMUSL, reason="MUSL allows setting invalid LC_CTYPE." + ), + ), + locale.LC_TIME, + ), +) def test_can_set_locale_invalid_set(lc_var): # Cannot set an invalid locale. before_locale = _get_current_locale(lc_var) diff --git a/pandas/tests/tslibs/test_parsing.py b/pandas/tests/tslibs/test_parsing.py index 6500afdf87beb..d514acfa1b685 100644 --- a/pandas/tests/tslibs/test_parsing.py +++ b/pandas/tests/tslibs/test_parsing.py @@ -25,7 +25,7 @@ def test_parsing_tzlocal_deprecated(): msg = "Pass the 'tz' keyword or call tz_localize after construction instead" dtstr = "Jan 15 2004 03:00 EST" - with tm.set_timezone("US/Eastern"): + with tm.set_timezone("EST5EDT"): with tm.assert_produces_warning(FutureWarning, match=msg): res, _ = parse_datetime_string_with_reso(dtstr) From 0b6d0632d6cdc2452a46c9b3298953439ba0cf13 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 27 Apr 2023 16:50:13 -0700 Subject: [PATCH 6/8] Correct skips for MUSL --- .github/workflows/unit-tests.yml | 7 +++++-- pandas/tests/config/test_localization.py | 6 +++--- pandas/tests/tslibs/test_parsing.py | 11 +++++++++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 08f42a27babed..3c5e776d006c2 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -265,7 +265,7 @@ jobs: export MUSL_LOCPATH=/usr/share/i18n/locales/musl apk update apk add musl-locales musl-locales-lang - - name: Build environment and Run Tests + - name: Build environment run: | /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate @@ -273,7 +273,10 @@ jobs: python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1 python setup.py build_ext -q -j$(nproc) python -m pip install --no-cache-dir --no-build-isolation --no-use-pep517 -e . - python -m pip list + python -m pip list --no-cache-dir + + - name: Run Tests + . ~/virtualenvs/pandas-dev/bin/activate export PANDAS_CI=1 python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml concurrency: diff --git a/pandas/tests/config/test_localization.py b/pandas/tests/config/test_localization.py index 8ea3085116eaa..3907f557d1075 100644 --- a/pandas/tests/config/test_localization.py +++ b/pandas/tests/config/test_localization.py @@ -52,13 +52,13 @@ def test_can_set_locale_valid_set(lc_var): "lc_var", ( locale.LC_ALL, + locale.LC_CTYPE, pytest.param( - locale.LC_CTYPE, + locale.LC_TIME, marks=pytest.mark.skipif( - ISMUSL, reason="MUSL allows setting invalid LC_CTYPE." + ISMUSL, reason="MUSL allows setting invalid LC_TIME." ), ), - locale.LC_TIME, ), ) def test_can_set_locale_invalid_set(lc_var): diff --git a/pandas/tests/tslibs/test_parsing.py b/pandas/tests/tslibs/test_parsing.py index d514acfa1b685..587527c2058d7 100644 --- a/pandas/tests/tslibs/test_parsing.py +++ b/pandas/tests/tslibs/test_parsing.py @@ -14,18 +14,25 @@ strptime, ) from pandas._libs.tslibs.parsing import parse_datetime_string_with_reso +from pandas.compat import ( + ISMUSL, + is_platform_windows, +) import pandas.util._test_decorators as td import pandas._testing as tm -@td.skip_if_windows +@pytest.mark.skipif( + is_platform_windows() or ISMUSL, + reason="TZ setting incorrect on Windows and MUSL Linux", +) def test_parsing_tzlocal_deprecated(): # GH#50791 msg = "Pass the 'tz' keyword or call tz_localize after construction instead" dtstr = "Jan 15 2004 03:00 EST" - with tm.set_timezone("EST5EDT"): + with tm.set_timezone("US/Eastern"): with tm.assert_produces_warning(FutureWarning, match=msg): res, _ = parse_datetime_string_with_reso(dtstr) From 1beb2adaced57ffd35fd7e59ded0262ab90a4925 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 27 Apr 2023 18:10:22 -0700 Subject: [PATCH 7/8] Do less stuff during install --- .github/workflows/unit-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3c5e776d006c2..cdf0666d08185 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -262,9 +262,8 @@ jobs: fi - name: Configure System Packages run: | - export MUSL_LOCPATH=/usr/share/i18n/locales/musl apk update - apk add musl-locales musl-locales-lang + apk add musl-locales - name: Build environment run: | /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev From 92a73aa9f603e31af1895c69becddd933b89a391 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 27 Apr 2023 19:24:36 -0700 Subject: [PATCH 8/8] syntax error --- .github/workflows/unit-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index cdf0666d08185..553d3e7d568a1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -275,6 +275,7 @@ jobs: python -m pip list --no-cache-dir - name: Run Tests + run: | . ~/virtualenvs/pandas-dev/bin/activate export PANDAS_CI=1 python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml