From 2ea33b253d7486bdab59160b8362616cb61fcf18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 11:41:45 +0100 Subject: [PATCH 01/10] Test more on GitHub Actions The only environments left on AppVeyor are now 3.5 and 3.6, which aren't supported on GitHub Actions. The remaining Python versions are now also tested on Windows using GitHub Actions. Additionally, a new group of dependencies called "oldest" is now tested as well to ensure compatibility with the oldest pinned versions in requirements.txt and setup.py. --- .appveyor.yml | 10 ------ .github/workflows/python-tox.yml | 57 +++++++++++++++++++++++++++++--- requirements-install.sh | 15 --------- requirements-oldest.txt | 27 +++++++++++++++ requirements-optional.txt | 6 ++-- setup.py | 12 +++---- tox.ini | 8 ++--- 7 files changed, 92 insertions(+), 43 deletions(-) delete mode 100755 requirements-install.sh create mode 100644 requirements-oldest.txt diff --git a/.appveyor.yml b/.appveyor.yml index 8af60b9b..7b79aae6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,20 +3,10 @@ environment: global: PATH: "C:\\Python27\\Scripts\\;%PATH%" matrix: - - TOXENV: py27-base - - TOXENV: py27-optional - TOXENV: py35-base - TOXENV: py35-optional - TOXENV: py36-base - TOXENV: py36-optional - - TOXENV: py37-base - - TOXENV: py37-optional - - TOXENV: py38-base - - TOXENV: py38-optional - - TOXENV: py39-base - - TOXENV: py39-optional - - TOXENV: py310-base - - TOXENV: py310-optional install: - git submodule update --init --recursive diff --git a/.github/workflows/python-tox.yml b/.github/workflows/python-tox.yml index 0e3e46db..7da64172 100644 --- a/.github/workflows/python-tox.yml +++ b/.github/workflows/python-tox.yml @@ -4,21 +4,68 @@ jobs: # Prevent duplicate builds for 'internal' pull requests on existing commits # Credit: https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012 if: github.event.push || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python: ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-2.7", "pypy-3.8"] + python: ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-latest, windows-latest] + deps: [base, optional] + include: + - python: "pypy-2.7" + os: ubuntu-latest + deps: base + - python: "pypy-3.8" + os: ubuntu-latest + deps: base + - python: "2.7" + os: ubuntu-latest + deps: oldest + - python: "3.7" + os: ubuntu-latest + deps: oldest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 with: submodules: true - - uses: actions/setup-python@v4 + - if: ${{ matrix.deps == 'base' }} + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} cache: pip - cache-dependency-path: "requirements*.txt" + cache-dependency-path: | + requirements.txt + requirements-test.txt + - if: ${{ matrix.deps == 'optional' }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: | + requirements.txt + requirements-optional.txt + requirements-test.txt + - if: ${{ matrix.deps == 'oldest' }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: | + requirements-oldest.txt + - name: Determine Python version for Tox + run: | + V=${{ matrix.python }} + if [[ "$V" = pypy-2* ]]; then + V=pypy + elif [[ "$V" = pypy-3* ]]; then + V=pypy3 + elif [[ "$V" = ~* ]]; then + V=py$(NORM_V=${V:1:4};echo ${NORM_V//./}) + else + V=py${V//./} + fi + echo TOX_PYTHON=$V >>$GITHUB_ENV - run: pip install tox - - run: tox -e py + run: tox -e ${{ env.TOX_PYTHON }}-${{ matrix.deps }} - if: ${{ always() }} run: python debug-info.py diff --git a/requirements-install.sh b/requirements-install.sh deleted file mode 100755 index b7a8d96d..00000000 --- a/requirements-install.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -ex - -if [[ $SIX_VERSION ]]; then - pip install six==$SIX_VERSION -fi - -pip install -r requirements-test.txt - -if [[ $USE_OPTIONAL == "true" ]]; then - pip install -r requirements-optional.txt -fi - -if [[ $CI == "true" ]]; then - pip install codecov -fi diff --git a/requirements-oldest.txt b/requirements-oldest.txt new file mode 100644 index 00000000..d24fdc33 --- /dev/null +++ b/requirements-oldest.txt @@ -0,0 +1,27 @@ +# This allows us to install the actually oldest supported dependencies and test whether that works. + +# requirements.txt +six==1.9 +webencodings==0.5.1 + +# requirements-optional.txt +genshi==0.7.1 ; python_version < '3.8' +genshi==0.7.6 ; python_version >= '3.8' +chardet==2.2.1 +lxml==3.4.0 ; python_version < '3.7' +# minimums for 3.x are actually different: +# - 3.7 is actually 4.1.1 +# - 3.8 is actually 4.3.5 +# - 3.9-3.10 is actually 4.5.2 +# - 3.11 is actually 4.9.0 +lxml==4.9.0 ; python_version >= '3.7' + +# requirements-test.txt +tox==3.15.1 +flake8==3.8.1 +pytest==4.6.10 ; python_version < '3' +pytest==5.4.2 ; python_version >= '3' +coverage==5.1 +pytest-expect==1.1.0 +mock==3.0.5 ; python_version < '3.6' +mock==4.0.2 ; python_version >= '3.6' \ No newline at end of file diff --git a/requirements-optional.txt b/requirements-optional.txt index 2e78c952..2e112e95 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -2,12 +2,12 @@ # We support a Genshi treewalker that can be used to serialize Genshi # streams. -genshi +genshi>=0.7.1 # chardet can be used as a fallback in case we are unable to determine # the encoding of a document. -chardet>=2.2 +chardet>=2.2.1 # lxml is supported with its own treebuilder ("lxml") and otherwise # uses the standard ElementTree support -lxml ; platform_python_implementation == 'CPython' +lxml>=3.4.0 ; platform_python_implementation == 'CPython' diff --git a/setup.py b/setup.py index b4c11811..30ee0575 100644 --- a/setup.py +++ b/setup.py @@ -108,23 +108,23 @@ def default_environment(): packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), install_requires=[ 'six>=1.9', - 'webencodings', + 'webencodings>=0.5.1', ], python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", extras_require={ # A conditional extra will only install these items when the extra is # requested and the condition matches. - "lxml:platform_python_implementation == 'CPython'": ["lxml"], + "lxml:platform_python_implementation == 'CPython'": ["lxml>=3.4.0"], # Standard extras, will be installed when the extra is requested. - "genshi": ["genshi"], - "chardet": ["chardet>=2.2"], + "genshi": ["genshi>=0.7.1"], + "chardet": ["chardet>=2.2.1"], # The all extra combines a standard extra which will be used anytime # the all extra is requested, and it extends it with a conditional # extra that will be installed whenever the condition matches and the # all extra is requested. - "all": ["genshi", "chardet>=2.2"], - "all:platform_python_implementation == 'CPython'": ["lxml"], + "all": ["genshi>=0.7.1", "chardet>=2.2.1"], + "all:platform_python_implementation == 'CPython'": ["lxml>=3.4.0"], }, ) diff --git a/tox.ini b/tox.ini index 42790f48..8229f9d3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,17 +1,17 @@ [tox] -envlist = py{27,35,36,37,38,39,310,311,py,py3}-{base,six19,optional} +envlist = py{27,35,36,37,38,39,310,311,py,py3}-{base,optional,oldest} [testenv] deps = - optional: -r{toxinidir}/requirements-optional.txt - -r{toxinidir}/requirements-test.txt + base: -r{toxinidir}/requirements-test.txt + optional: -r{toxinidir}/requirements-test.txt -r{toxinidir}/requirements-optional.txt + oldest: -r{toxinidir}/requirements-oldest.txt doc: Sphinx passenv = PYTEST_COMMAND # this is maintained so one can, e.g., PYTEST_COMMAND="coverage run -m pytest" COVERAGE_RUN_OPTIONS commands = - six19: pip install six==1.9 {env:PYTEST_COMMAND:{envbindir}/pytest} {posargs} flake8 {toxinidir} From 5b2340b7d16bc3b251b083964ddac64ce3528bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 11:47:05 +0100 Subject: [PATCH 02/10] Fix GHA file --- .github/workflows/python-tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tox.yml b/.github/workflows/python-tox.yml index 7da64172..8575d7c0 100644 --- a/.github/workflows/python-tox.yml +++ b/.github/workflows/python-tox.yml @@ -66,6 +66,6 @@ jobs: fi echo TOX_PYTHON=$V >>$GITHUB_ENV - run: pip install tox - run: tox -e ${{ env.TOX_PYTHON }}-${{ matrix.deps }} + - run: tox -e ${{ env.TOX_PYTHON }}-${{ matrix.deps }} - if: ${{ always() }} run: python debug-info.py From e45d6d3f03eae1c168ede6afadaf1d4361f8b661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 11:49:50 +0100 Subject: [PATCH 03/10] Fix tox.ini --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8229f9d3..fb228e96 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,8 @@ envlist = py{27,35,36,37,38,39,310,311,py,py3}-{base,optional,oldest} [testenv] deps = base: -r{toxinidir}/requirements-test.txt - optional: -r{toxinidir}/requirements-test.txt -r{toxinidir}/requirements-optional.txt + optional: -r{toxinidir}/requirements-test.txt + optional: -r{toxinidir}/requirements-optional.txt oldest: -r{toxinidir}/requirements-oldest.txt doc: Sphinx From 138662b384ae507b0b549700804c6eeda421536b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 11:51:46 +0100 Subject: [PATCH 04/10] Remove Tox from test deps --- requirements-oldest.txt | 1 - requirements-test.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements-oldest.txt b/requirements-oldest.txt index d24fdc33..06509831 100644 --- a/requirements-oldest.txt +++ b/requirements-oldest.txt @@ -17,7 +17,6 @@ lxml==3.4.0 ; python_version < '3.7' lxml==4.9.0 ; python_version >= '3.7' # requirements-test.txt -tox==3.15.1 flake8==3.8.1 pytest==4.6.10 ; python_version < '3' pytest==5.4.2 ; python_version >= '3' diff --git a/requirements-test.txt b/requirements-test.txt index 8c0ca7c7..cccb5691 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ -r requirements.txt -tox>=3.15.1,<4 flake8>=3.8.1,<6 pytest>=4.6.10,<5 ; python_version < '3' pytest>=5.4.2,<7 ; python_version >= '3' From 37d21ebb0f164cbb0e5e300d11f788a522863c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 12:41:48 +0100 Subject: [PATCH 05/10] Cross-platform Tox version mangling --- .github/workflows/python-tox.yml | 21 +++++++------------ toxver.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 14 deletions(-) create mode 100755 toxver.py diff --git a/.github/workflows/python-tox.yml b/.github/workflows/python-tox.yml index 8575d7c0..15d92ce5 100644 --- a/.github/workflows/python-tox.yml +++ b/.github/workflows/python-tox.yml @@ -52,20 +52,13 @@ jobs: cache: pip cache-dependency-path: | requirements-oldest.txt - - name: Determine Python version for Tox - run: | - V=${{ matrix.python }} - if [[ "$V" = pypy-2* ]]; then - V=pypy - elif [[ "$V" = pypy-3* ]]; then - V=pypy3 - elif [[ "$V" = ~* ]]; then - V=py$(NORM_V=${V:1:4};echo ${NORM_V//./}) - else - V=py${V//./} - fi - echo TOX_PYTHON=$V >>$GITHUB_ENV + - if: ${{ matrix.os == 'windows-latest' }} + name: Determine environment name for Tox (PowerShell) + run: python toxver.py >> $env:GITHUB_ENV + - if: ${{ matrix.os == 'ubuntu-latest' }} + name: Determine environment name for Tox (Bash) + run: python toxver.py >> $GITHUB_ENV - run: pip install tox - - run: tox -e ${{ env.TOX_PYTHON }}-${{ matrix.deps }} + - run: tox - if: ${{ always() }} run: python debug-info.py diff --git a/toxver.py b/toxver.py new file mode 100755 index 00000000..6094ed8e --- /dev/null +++ b/toxver.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import sys + + +def main(argv): + if len(argv) != 3: + print("usage: toxver.py [python-version] [deps]", file=sys.stderr) + return 1 + + deps = argv[2] + + if argv[1].startswith("pypy-2"): + print("TOXENV=pypy-" + deps) + return 0 + + if argv[1].startswith("pypy-3"): + print("TOXENV=pypy3-" + deps) + return 0 + + if argv[1].startswith("~"): + ver = argv[1][1:5] + else: + ver = argv[1] + + ver = ver.replace(".", "") + print("TOXENV=py" + ver + "-" + deps) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 2f568a0aca5452870b029427420f07d1d7922275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 12:44:32 +0100 Subject: [PATCH 06/10] Use toxver.py correctly --- .github/workflows/python-tox.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tox.yml b/.github/workflows/python-tox.yml index 15d92ce5..53c2a89b 100644 --- a/.github/workflows/python-tox.yml +++ b/.github/workflows/python-tox.yml @@ -54,10 +54,10 @@ jobs: requirements-oldest.txt - if: ${{ matrix.os == 'windows-latest' }} name: Determine environment name for Tox (PowerShell) - run: python toxver.py >> $env:GITHUB_ENV + run: python toxver.py ${{ matrix.python }} ${{ matrix.deps }} >> $env:GITHUB_ENV - if: ${{ matrix.os == 'ubuntu-latest' }} name: Determine environment name for Tox (Bash) - run: python toxver.py >> $GITHUB_ENV + run: python toxver.py ${{ matrix.python }} ${{ matrix.deps }} >> $GITHUB_ENV - run: pip install tox - run: tox - if: ${{ always() }} From ee25b456b0ad8e672192c8a044d48c608dd6312f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 12:51:59 +0100 Subject: [PATCH 07/10] Don't run 2.7 on Windows on GHA (pip cache crashes) --- .appveyor.yml | 2 ++ .github/workflows/python-tox.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 7b79aae6..7661aa63 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,6 +3,8 @@ environment: global: PATH: "C:\\Python27\\Scripts\\;%PATH%" matrix: + - TOXENV: py27-base + - TOXENV: py27-optional - TOXENV: py35-base - TOXENV: py35-optional - TOXENV: py36-base diff --git a/.github/workflows/python-tox.yml b/.github/workflows/python-tox.yml index 53c2a89b..cfcc42e6 100644 --- a/.github/workflows/python-tox.yml +++ b/.github/workflows/python-tox.yml @@ -7,7 +7,8 @@ jobs: strategy: fail-fast: false matrix: - python: ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"] + # 2.7, 3.5, and 3.6 run on Windows via AppVeyor + python: ["3.7", "3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest, windows-latest] deps: [base, optional] include: From 7e0239baedf7707f7b99fca76b74be3165873806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 12:58:31 +0100 Subject: [PATCH 08/10] Try flake8 3.9.2 as 3.8.x was crashing on Python 3.7 --- requirements-oldest.txt | 6 ++++-- requirements-test.txt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements-oldest.txt b/requirements-oldest.txt index 06509831..400e4b9c 100644 --- a/requirements-oldest.txt +++ b/requirements-oldest.txt @@ -8,7 +8,9 @@ webencodings==0.5.1 genshi==0.7.1 ; python_version < '3.8' genshi==0.7.6 ; python_version >= '3.8' chardet==2.2.1 -lxml==3.4.0 ; python_version < '3.7' +# this should be 3.4.0 but there are no Linux +# binary wheels for older releases +lxml==3.8.0 ; python_version < '3.7' # minimums for 3.x are actually different: # - 3.7 is actually 4.1.1 # - 3.8 is actually 4.3.5 @@ -17,7 +19,7 @@ lxml==3.4.0 ; python_version < '3.7' lxml==4.9.0 ; python_version >= '3.7' # requirements-test.txt -flake8==3.8.1 +flake8==3.9.2 pytest==4.6.10 ; python_version < '3' pytest==5.4.2 ; python_version >= '3' coverage==5.1 diff --git a/requirements-test.txt b/requirements-test.txt index cccb5691..ededf9cb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ -r requirements.txt -flake8>=3.8.1,<6 +flake8>=3.9.2,<6 pytest>=4.6.10,<5 ; python_version < '3' pytest>=5.4.2,<7 ; python_version >= '3' coverage>=5.1,<6 From 9382b10bd5d7d7d5304e302ca4d0835df2c74688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 13:07:09 +0100 Subject: [PATCH 09/10] Split flake8 deps to 2.7-3.5 and 3.6+ --- requirements-oldest.txt | 3 ++- requirements-test.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements-oldest.txt b/requirements-oldest.txt index 400e4b9c..68d0f13d 100644 --- a/requirements-oldest.txt +++ b/requirements-oldest.txt @@ -19,7 +19,8 @@ lxml==3.8.0 ; python_version < '3.7' lxml==4.9.0 ; python_version >= '3.7' # requirements-test.txt -flake8==3.9.2 +flake8==3.9.2 ; python_version < '3.6' +flake8==5.0.4; python_version >= '3.6' pytest==4.6.10 ; python_version < '3' pytest==5.4.2 ; python_version >= '3' coverage==5.1 diff --git a/requirements-test.txt b/requirements-test.txt index ededf9cb..27866e59 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,7 @@ -r requirements.txt -flake8>=3.9.2,<6 +flake8==3.9.2 ; python_version < '3.6' +flake8>=5.0.4; python_version >= '3.6' pytest>=4.6.10,<5 ; python_version < '3' pytest>=5.4.2,<7 ; python_version >= '3' coverage>=5.1,<6 From 3c05d2dc9044d73615aae2957ef3de1552f6e0eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Mar 2023 13:51:53 +0100 Subject: [PATCH 10/10] Document toxver.py --- toxver.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/toxver.py b/toxver.py index 6094ed8e..68eb71ec 100755 --- a/toxver.py +++ b/toxver.py @@ -1,4 +1,25 @@ #!/usr/bin/env python + +""" +usage: toxver.py [python-version] [deps] + +Returns a Tox environment name given a GHA matrix Python version and dependencies. +Many GHA configurations do this with inline Bash scripts but we want our solution +to be cross-platform and work on Windows workers, too. + +Examples: + + $ toxver.py pypy-3.8 base + TOXENV=pypy3-base + + $ toxver.py 2.7 oldest + TOXENV=py27-oldest + + $ toxver.py ~3.12.0-0 optional + TOXENV=py312-optional + +""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -9,7 +30,7 @@ def main(argv): if len(argv) != 3: - print("usage: toxver.py [python-version] [deps]", file=sys.stderr) + print(__doc__.strip(), file=sys.stderr) return 1 deps = argv[2]