From b4f497c05fc4a04c75ed85aaacba2b452ca3004c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 21 Dec 2022 16:58:19 +0100 Subject: [PATCH] Do not use packaging.LegacyVersion #3171 #3177 * Switch to using packvers fork of packaging * Bump pip requirements parser, but it still using packaging. * Bump dparse2 which is using packvers * Pin SPDX tools for cope with breaking APIs * Update release script to build one release wheel for each Python version that each contain a pickle of the License index * Streamline the app build * Bump version and use correct SPDX version Signed-off-by: Philippe Ombredanne --- .github/workflows/scancode-release.yml | 171 ++++++++---------- CHANGELOG.rst | 14 ++ README.rst | 2 +- azure-pipelines.yml | 34 ++-- ...-dist.sh => scancode-create-pypi-sdist.sh} | 9 +- etc/release/scancode-create-pypi-wheel.sh | 39 ++++ .../scancode-create-release-app-linux.sh | 31 +++- .../scancode-create-release-app-macos.sh | 29 ++- .../scancode-create-release-app-sources.sh | 11 +- .../scancode-create-release-app-windows.sh | 39 ++-- etc/scripts/gen_pypi_simple.py | 6 +- etc/scripts/requirements.txt | 3 +- etc/scripts/utils_dejacode.py | 2 +- etc/scripts/utils_pip_compatibility_tags.py | 2 +- etc/scripts/utils_thirdparty.py | 4 +- requirements.txt | 5 +- setup-mini.cfg | 14 +- setup.cfg | 12 +- src/packagedcode/pypi.py | 10 +- src/scancode/outdated.py | 15 +- src/scancode_config.py | 4 +- tests/licensedcode/test_zzzz_cache.py | 8 +- .../invalid_spec/output.expected.json | 24 +-- tests/scancode/test_outdated.py | 4 +- 24 files changed, 270 insertions(+), 222 deletions(-) rename etc/release/{scancode-create-pypi-dist.sh => scancode-create-pypi-sdist.sh} (73%) create mode 100755 etc/release/scancode-create-pypi-wheel.sh diff --git a/.github/workflows/scancode-release.yml b/.github/workflows/scancode-release.yml index a01fbc570ff..60dca1ab48e 100644 --- a/.github/workflows/scancode-release.yml +++ b/.github/workflows/scancode-release.yml @@ -20,11 +20,11 @@ on: permissions: {} jobs: - build_scancode_for_pypi: + build_scancode_wheel_for_pypi: permissions: contents: read # to fetch code (actions/checkout) - name: Build PyPI archives + name: Build PyPI wheels runs-on: ubuntu-20.04 defaults: @@ -33,51 +33,65 @@ jobs: strategy: fail-fast: true + matrix: + pyver: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 - - name: Set up Python + - name: Set up Python on ${{ matrix.pyver }} uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ matrix.pyver }} - - name: Install requirements then build main and mini wheel and sdist - run: etc/release/scancode-create-pypi-dist.sh + - name: Install requirements then build main and mini wheel + run: etc/release/scancode-create-pypi-wheel.sh - - name: Collect built main wheel + - name: Collect built wheels for ${{ matrix.pyver }} uses: actions/upload-artifact@v3 with: - name: main_wheel - path: dist/scancode_toolkit-3*.whl + name: wheels-${{ matrix.pyver }} + path: dist/*.whl - - name: Collect built main sdist - uses: actions/upload-artifact@v3 - with: - name: main_sdist - path: dist/scancode-toolkit-3*.tar.gz - - name: Collect built mini wheel - uses: actions/upload-artifact@v3 + build_scancode_sdist_for_pypi: + permissions: + contents: read # to fetch code (actions/checkout) + + name: Build PyPI sdist archives + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 with: - name: mini_wheel - path: dist/scancode_toolkit_mini-3*.whl + python-version: "3.10" - - name: Collect built mini sdist + - name: Install requirements then build main and mini sdist + run: etc/release/scancode-create-pypi-sdist.sh + + - name: Collect built sdist uses: actions/upload-artifact@v3 with: - name: mini_sdist - path: dist/scancode-toolkit-mini*.tar.gz + name: sdists + path: dist/*.tar.gz build_scancode_for_release_linux: permissions: contents: read # to fetch code (actions/checkout) - name: Build Release for linux + name: Build app Release for linux runs-on: ubuntu-20.04 - needs: - - build_scancode_for_pypi defaults: run: @@ -92,18 +106,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 - - - name: Download a the main wheel - uses: actions/download-artifact@v3 - with: - name: main_wheel - path: dist + python-version: "3.8" - name: Build linux app archive - run: | - ./configure --rel - etc/release/scancode-create-release-app-linux.sh + run: etc/release/scancode-create-release-app-linux.sh - name: Collect built linux app uses: actions/upload-artifact@v3 @@ -116,10 +122,8 @@ jobs: permissions: contents: read # to fetch code (actions/checkout) - name: Build Release for mac + name: Build app Release for mac runs-on: ubuntu-20.04 - needs: - - build_scancode_for_pypi defaults: run: @@ -134,18 +138,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 - - - name: Download a the main wheel - uses: actions/download-artifact@v3 - with: - name: main_wheel - path: dist + python-version: "3.8" - name: Build mac app archive - run: | - ./configure --rel - etc/release/scancode-create-release-app-macos.sh + run: etc/release/scancode-create-release-app-macos.sh - name: Collect built mac app uses: actions/upload-artifact@v3 @@ -158,10 +154,8 @@ jobs: permissions: contents: read # to fetch code (actions/checkout) - name: Build Release for windows + name: Build app Release for windows runs-on: ubuntu-20.04 - needs: - - build_scancode_for_pypi defaults: run: @@ -175,18 +169,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 - - - name: Download a the main wheel - uses: actions/download-artifact@v3 - with: - name: main_wheel - path: dist + python-version: "3.8" - name: Build windows app archive - run: | - ./configure --rel - etc/release/scancode-create-release-app-windows.sh + run: etc/release/scancode-create-release-app-windows.sh - name: Collect built windows app uses: actions/upload-artifact@v3 @@ -199,10 +185,8 @@ jobs: permissions: contents: read # to fetch code (actions/checkout) - name: Build source + name: Build app source runs-on: ubuntu-20.04 - needs: - - build_scancode_for_pypi defaults: run: @@ -216,18 +200,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 - - - name: Download a the main sdist - uses: actions/download-artifact@v3 - with: - name: main_sdist - path: dist + python-version: "3.10" - name: Build source archive with deps - run: | - ./configure --rel - etc/release/scancode-create-release-app-sources.sh + run: etc/release/scancode-create-release-app-sources.sh - name: Collect built source app tarball uses: actions/upload-artifact@v3 @@ -236,13 +212,13 @@ jobs: path: release/* - smoke_test_install_and_run_pypi_dists_posix: + smoke_test_install_and_run_pypi_wheels_on_posix: permissions: contents: read # to fetch code (actions/checkout) - name: Test POSIX PyPI wheels + name: Test PyPI wheels on linux and mac needs: - - build_scancode_for_pypi + - build_scancode_wheel_for_pypi runs-on: ${{ matrix.os }} defaults: @@ -252,10 +228,8 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-22.04, macos-12] - pyver: ["3.9", "3.10"] - # os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, macos-10.15, macos-11, macos-12] - # pyver: ["3.7", "3.8", "3.9", "3.10"] + os: [ubuntu-20.04, ubuntu-22.04, macos-11, macos-12] + pyver: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -265,10 +239,10 @@ jobs: with: python-version: ${{ matrix.pyver }} - - name: Download a single artifact mainw + - name: Download wheels artifact uses: actions/download-artifact@v3 with: - name: main_wheel + name: wheels-${{ matrix.pyver }} path: dist - name: test install wheels @@ -277,21 +251,20 @@ jobs: rm -rf venv; \ python -m venv venv venv/bin/python -m pip install --upgrade pip wheel setuptools - for f in `find dist -type f`; \ + for f in `find dist -type f -name "*${{ matrix.pyver }}*"`; \ do \ venv/bin/python -m pip install --force-reinstall "$f[full]" ; \ venv/bin/scancode -clipeu --json-pp - some.file ; \ - venv/bin/python -m pip uninstall --yes scancode-toolkit ; \ done - smoke_test_install_and_run_pypi_dists_windows: + smoke_test_install_and_run_pypi_wheels_on_windows: permissions: contents: read # to fetch code (actions/checkout) - name: Test Windows PyPI wheels + name: Test PyPI wheels on Windows needs: - - build_scancode_for_pypi + - build_scancode_wheel_for_pypi runs-on: ${{ matrix.os }} defaults: @@ -301,10 +274,8 @@ jobs: strategy: fail-fast: true matrix: - os: [windows-2022] - pyver: ["3.9", "3.10"] - #os: [windows-2019, windows-2022] - #pyver: ["3.7", "3.8", "3.9", "3.10"] + os: [windows-2019, windows-2022] + pyver: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -314,10 +285,10 @@ jobs: with: python-version: ${{ matrix.pyver }} - - name: Download a single artifact mainw + - name: Download wheels artifact uses: actions/download-artifact@v3 with: - name: main_wheel + name: wheels-${{ matrix.pyver }} path: dist - name: test install wheel @@ -325,11 +296,10 @@ jobs: echo "license: gpl-2.0" > some.file python -m venv venv venv/Scripts/python.exe -m pip install --upgrade pip - for f in `find dist -type f`; \ + for f in `find dist -type f -name "*${{ matrix.pyver }}*"`; \ do \ venv/Scripts/python.exe -m pip install --force-reinstall "$f[full]" ; \ venv/Scripts/scancode -clipeu --json-pp - some.file ; \ - venv/Scripts/python.exe -m pip uninstall --yes scancode-toolkit ; \ done @@ -390,7 +360,7 @@ jobs: strategy: fail-fast: true matrix: - os: [macos-10.15, macos-11, macos-12] + os: [macos-11, macos-12] pyver: [3.8] steps: @@ -509,9 +479,10 @@ jobs: publish_to_pypi: name: Publish to PyPI needs: - - build_scancode_for_pypi - - smoke_test_install_and_run_pypi_dists_windows - - smoke_test_install_and_run_pypi_dists_posix + - build_scancode_wheel_for_pypi + - build_scancode_sdist_for_pypi + - smoke_test_install_and_run_pypi_wheels_on_windows + - smoke_test_install_and_run_pypi_wheels_on_posix - publish_to_gh_release runs-on: ubuntu-20.04 defaults: @@ -520,7 +491,7 @@ jobs: strategy: fail-fast: true matrix: - dist_names: [main_wheel, main_sdist, mini_wheel, mini_sdist] + dist_names: ["wheels-3.7", "wheels-3.8", "wheels-3.9", "wheels-3.10", sdists] steps: - name: Set up Python diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9b17cb97a70..dcae98079b2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -54,6 +54,20 @@ License detection: detected license only once. This data can contain the reference license text as an option. + +v31.2.3 - 2022-12-24 +---------------------------------- + +This is a minor bugfix release. + +There is a fix for an installation issue with the new "packaging" version 22.0. +This is replaced by a fork named "packvers" to work around +https://github.com/pypa/packaging/issues/530 + +We also improved the compatibility for pre-built wheels and now build one +wheel for each Python version to work around some Python pickle bug. + + v31.2.1 - 2022-10-05 ---------------------------------- diff --git a/README.rst b/README.rst index 49ece2feb9e..0e1fa3c9aea 100644 --- a/README.rst +++ b/README.rst @@ -134,7 +134,7 @@ Installation Before installing ScanCode make sure that you have installed the prerequisites properly. This means installing Python 3.8 for x86/64 architectures. -We support Python 3.7, 3.8, 3.9 and 3.10. +We support Python 3.8, 3.9 and 3.10. See `prerequisites `_ for detailed information on the support platforms and Python versions. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a2962b9a720..4de4f073ff5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,14 +8,14 @@ jobs: ################################################################################ -# These jobs are using VMs and Azure-provided Pythons 3.8 +# These jobs are using VMs and Azure-provided Pythons 3.10 ################################################################################ - template: etc/ci/azure-posix.yml parameters: job_name: core_tests image_name: ubuntu-20.04 - python_versions: ['3.8'] + python_versions: ['3.10'] test_suites: misc_and_scancode: | # cli tests are launched below on all OSes @@ -82,7 +82,7 @@ jobs: parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] python_architecture: x64 test_suites: all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -91,7 +91,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] python_architecture: x64 test_suites: all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -100,7 +100,7 @@ jobs: parameters: job_name: macos1015_cpython_1 image_name: macos-10.15 - python_versions: ['3.7', '3.8'] + python_versions: ['3.8'] python_architecture: x64 test_suites: all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -118,7 +118,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] python_architecture: x64 test_suites: all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -127,7 +127,7 @@ jobs: parameters: job_name: macos12_cpython image_name: macos-12 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] python_architecture: x64 test_suites: all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -136,7 +136,7 @@ jobs: parameters: job_name: win2019_cpython_1 image_name: windows-2019 - python_versions: ['3.7', '3.8'] + python_versions: ['3.8'] python_architecture: x64 test_suites: all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py @@ -154,7 +154,7 @@ jobs: parameters: job_name: win2022_cpython_1 image_name: windows-2022 - python_versions: ['3.7', '3.8'] + python_versions: ['3.8'] python_architecture: x64 test_suites: all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py @@ -178,7 +178,7 @@ jobs: parameters: job_name: ubuntu20_test_all_supported_click_versions image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] python_architecture: x64 test_suites: click_versions: | @@ -199,7 +199,7 @@ jobs: parameters: job_name: ubuntu18_cpython_latest_from_pip image_name: ubuntu-18.04 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -207,7 +207,7 @@ jobs: parameters: job_name: ubuntu20_cpython_latest_from_pip image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -215,7 +215,7 @@ jobs: parameters: job_name: macos1015_cpython_latest_from_pip image_name: macos-10.15 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -223,7 +223,7 @@ jobs: parameters: job_name: macos11_cpython_latest_from_pip image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -231,7 +231,7 @@ jobs: parameters: job_name: macos12_cpython_latest_from_pip image_name: macos-12 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py @@ -239,7 +239,7 @@ jobs: parameters: job_name: win2019_cpython_latest_from_pip image_name: windows-2019 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py @@ -247,7 +247,7 @@ jobs: parameters: job_name: win2022_cpython_latest_from_pip image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py diff --git a/etc/release/scancode-create-pypi-dist.sh b/etc/release/scancode-create-pypi-sdist.sh similarity index 73% rename from etc/release/scancode-create-pypi-dist.sh rename to etc/release/scancode-create-pypi-sdist.sh index c46c650784f..fe8cdd75440 100755 --- a/etc/release/scancode-create-pypi-dist.sh +++ b/etc/release/scancode-create-pypi-sdist.sh @@ -9,7 +9,7 @@ # ################################################################################ -# ScanCode release build script for PyPI wheels and sdists +# ScanCode release build script for PyPI sdists ################################################################################ set -e @@ -17,14 +17,13 @@ set -e #set -x ./configure --dev -./scancode --reindex-licenses -venv/bin/python setup.py --quiet sdist bdist_wheel +venv/bin/python setup.py --quiet sdist -rm -rf build .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info +rm -rf build .eggs src/scancode_toolkit*.egg-info src/scancode_toolkit_mini*.egg-info cp setup.cfg setup-main.cfg cp setup-mini.cfg setup.cfg -venv/bin/python setup.py --quiet sdist bdist_wheel +venv/bin/python setup.py --quiet sdist cp setup-main.cfg setup.cfg rm setup-main.cfg diff --git a/etc/release/scancode-create-pypi-wheel.sh b/etc/release/scancode-create-pypi-wheel.sh new file mode 100755 index 00000000000..e047d7146a4 --- /dev/null +++ b/etc/release/scancode-create-pypi-wheel.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +################################################################################ +# ScanCode release build script for PyPI wheels. +# Build a wheel for the current Python version +################################################################################ + +set -e +# Un-comment to trace execution +#set -x + +./configure --dev +./scancode --reindex-licenses + +python_tag=$( python -c "import platform;print(f\"cp{''.join(platform.python_version_tuple()[:2])}\")" ) + +venv/bin/python setup.py --quiet bdist_wheel --python-tag $python_tag + +rm -rf build .eggs src/scancode_toolkit*.egg-info src/scancode_toolkit_mini*.egg-info +cp setup.cfg setup-main.cfg +cp setup-mini.cfg setup.cfg + +venv/bin/python setup.py --quiet bdist_wheel --python-tag $python_tag + +cp setup-main.cfg setup.cfg +rm setup-main.cfg + +venv/bin/twine check dist/* + +set +e +set +x diff --git a/etc/release/scancode-create-release-app-linux.sh b/etc/release/scancode-create-release-app-linux.sh index c68dfb4dd30..a6f97741fbc 100755 --- a/etc/release/scancode-create-release-app-linux.sh +++ b/etc/release/scancode-create-release-app-linux.sh @@ -22,21 +22,18 @@ python_dot_version=3.8 python_version=38 python_exe="python$python_dot_version" release_dir=scancode-toolkit-$(git describe --tags) + rm -rf $release_dir mkdir -p $release_dir echo -n "$python_exe" > $release_dir/PYTHON_EXECUTABLE git describe --tags > $release_dir/SCANCODE_VERSION thirdparty_dir=$release_dir/thirdparty +thirdparty_src_dir=$release_dir/thirdparty-src mkdir -p $thirdparty_dir +mkdir -p $thirdparty_src_dir -venv/bin/python etc/scripts/fetch_thirdparty.py \ - --requirements requirements-linux.txt \ - --requirements requirements.txt \ - --dest $thirdparty_dir \ - --operating-system=$operating_system \ - --python-version=$python_version \ - --wheels --use-cached-index +./configure --rel venv/bin/python etc/scripts/fetch_thirdparty.py \ --requirements requirements-native.txt \ @@ -46,12 +43,30 @@ venv/bin/python etc/scripts/fetch_thirdparty.py \ --wheel-only extractcode-7z \ --wheel-only extractcode-libarchive \ --wheel-only typecode-libmagic \ + --dest $thirdparty_src_dir \ + --sdists \ + --use-cached-index + +venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements requirements-linux.txt \ + --requirements requirements.txt \ --dest $thirdparty_dir \ - --sdists --use-cached-index + --operating-system=$operating_system \ + --python-version=$python_version \ + --wheels \ + --use-cached-index + +mv $thirdparty_src_dir/* $thirdparty_dir/ +rm -rf $thirdparty_src_dir mkdir -p $release_dir/etc cp -r etc/thirdparty $release_dir/etc +# Build the wheel +./configure --dev +./scancode --reindex-licenses +venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version + cp -r \ dist/scancode_*.whl \ scancode extractcode configure \ diff --git a/etc/release/scancode-create-release-app-macos.sh b/etc/release/scancode-create-release-app-macos.sh index bd650b8a067..49c48f1de07 100755 --- a/etc/release/scancode-create-release-app-macos.sh +++ b/etc/release/scancode-create-release-app-macos.sh @@ -22,20 +22,18 @@ python_dot_version=3.8 python_version=38 python_exe="python3" release_dir=scancode-toolkit-$(git describe --tags) + rm -rf $release_dir mkdir -p $release_dir echo -n "$python_exe" > $release_dir/PYTHON_EXECUTABLE git describe --tags > $release_dir/SCANCODE_VERSION thirdparty_dir=$release_dir/thirdparty +thirdparty_src_dir=$release_dir/thirdparty-src mkdir -p $thirdparty_dir +mkdir -p $thirdparty_src_dir -venv/bin/python etc/scripts/fetch_thirdparty.py \ - --requirements requirements.txt \ - --dest $thirdparty_dir \ - --operating-system=$operating_system \ - --python-version=$python_version \ - --wheels --use-cached-index +./configure --rel venv/bin/python etc/scripts/fetch_thirdparty.py \ --requirements requirements-native.txt \ @@ -44,12 +42,29 @@ venv/bin/python etc/scripts/fetch_thirdparty.py \ --wheel-only extractcode-7z \ --wheel-only extractcode-libarchive \ --wheel-only typecode-libmagic \ + --dest $thirdparty_src_dir \ + --sdists \ + --use-cached-index + +venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements requirements.txt \ --dest $thirdparty_dir \ - --sdists --use-cached-index + --operating-system=$operating_system \ + --python-version=$python_version \ + --wheels \ + --use-cached-index + +mv $thirdparty_src_dir/* $thirdparty_dir/ +rm -rf $thirdparty_src_dir mkdir -p $release_dir/etc cp -r etc/thirdparty $release_dir/etc +# Build the wheel +./configure --dev +./scancode --reindex-licenses +venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version + cp -r \ dist/scancode_*.whl \ scancode extractcode configure \ diff --git a/etc/release/scancode-create-release-app-sources.sh b/etc/release/scancode-create-release-app-sources.sh index e3201733542..1a73aa7485b 100755 --- a/etc/release/scancode-create-release-app-sources.sh +++ b/etc/release/scancode-create-release-app-sources.sh @@ -24,6 +24,13 @@ git describe --tags > $release_dir/SCANCODE_VERSION thirdparty_dir=$release_dir/thirdparty mkdir -p $thirdparty_dir +# build an sdist +./configure --dev +venv/bin/python setup.py --quiet sdist +mv dist/*.tar.gz $release_dir + +./configure --rel + venv/bin/python etc/scripts/fetch_thirdparty.py \ --requirements requirements.txt \ --requirements requirements-native.txt \ @@ -34,12 +41,12 @@ venv/bin/python etc/scripts/fetch_thirdparty.py \ --wheel-only extractcode-libarchive \ --wheel-only typecode-libmagic \ --dest $thirdparty_dir \ - --sdists --use-cached-index + --sdists \ + --use-cached-index mkdir -p $release_dir/etc cp -r etc/thirdparty $release_dir/etc -mv dist/*.tar.gz $release_dir cp -r \ scancode.bat scancode extractcode extractcode.bat configure configure.bat \ diff --git a/etc/release/scancode-create-release-app-windows.sh b/etc/release/scancode-create-release-app-windows.sh index b2e692f2596..9cefc9fd161 100755 --- a/etc/release/scancode-create-release-app-windows.sh +++ b/etc/release/scancode-create-release-app-windows.sh @@ -24,33 +24,48 @@ release_dir=scancode-toolkit-$(git describe --tags) rm -rf $release_dir mkdir -p $release_dir + echo -n "$python_exe" > $release_dir/PYTHON_EXECUTABLE git describe --tags > $release_dir/SCANCODE_VERSION thirdparty_dir=$release_dir/thirdparty +thirdparty_src_dir=$release_dir/thirdparty-src mkdir -p $thirdparty_dir +mkdir -p $thirdparty_src_dir -venv/bin/python etc/scripts/fetch_thirdparty.py \ - --requirements=requirements.txt \ - --dest $thirdparty_dir \ - --operating-system=$operating_system \ - --python-version=$python_version \ - --wheels --use-cached-index +./configure --rel venv/bin/python etc/scripts/fetch_thirdparty.py \ - --requirements=requirements-native.txt \ - --wheel-only extractcode \ + --requirements requirements-native.txt \ + --wheel-only packagedcode-msitools \ + --wheel-only rpm-inspector-rpm \ --wheel-only extractcode-7z \ --wheel-only extractcode-libarchive \ --wheel-only typecode-libmagic \ - --wheel-only packagedcode-msitools \ - --wheel-only rpm-inspector-rpm \ + --dest $thirdparty_src_dir \ + --sdists \ + --use-cached-index + +venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements requirements.txt \ --dest $thirdparty_dir \ - --sdists --use-cached-index + --operating-system=$operating_system \ + --python-version=$python_version \ + --wheels \ + --use-cached-index + +mv $thirdparty_src_dir/* $thirdparty_dir/ +rm -rf $thirdparty_src_dir -cp dist/scancode_*.whl $release_dir mkdir -p $release_dir/etc cp -r etc/thirdparty $release_dir/etc + +# Build the wheel +./configure --dev +./scancode --reindex-licenses +venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version + cp -r \ + dist/scancode_*.whl \ scancode.bat extractcode.bat configure.bat \ *.rst \ samples \ diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 03312ab3c67..214d90dc519 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -118,7 +118,7 @@ def build_per_package_index(pkg_name, packages, base_url): """ document.append(header) - for package in packages: + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ @@ -141,8 +141,8 @@ def build_links_package_index(packages_by_package_name, base_url): """ document.append(header) - for _name, packages in packages_by_package_name.items(): - for package in packages: + for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]): + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index ebb404b72d6..7c514da96aa 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -8,4 +8,5 @@ pip setuptools twine wheel -build \ No newline at end of file +build +packvers diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index f28e24797ab..c42e6c9313b 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -15,7 +15,7 @@ import requests import saneyaml -from packaging import version as packaging_version +from packvers import version as packaging_version """ Utility to create and retrieve package and ABOUT file data from DejaCode. diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 5d5eb34c887..af42a0cdd25 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,7 +27,7 @@ import re -from packaging.tags import ( +from packvers.tags import ( compatible_tags, cpython_tags, generic_tags, diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 120598e0d4e..ab8dde45d08 100755 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -28,8 +28,8 @@ from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name -from packaging import tags as packaging_tags -from packaging import version as packaging_version +from packvers import tags as packaging_tags +from packvers import version as packaging_version import utils_pip_compatibility_tags diff --git a/requirements.txt b/requirements.txt index 0034ed5749e..f741ab16e94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ container-inspector==31.1.0 cryptography==37.0.4 debian-inspector==31.0.0 dockerfile-parse==1.2.0 -dparse2==0.6.1 +dparse2==0.7.0 extractcode==31.0.0 extractcode-7z==16.5.210531 extractcode-libarchive==3.5.1.210531 @@ -41,10 +41,11 @@ more-itertools==8.13.0 normality==2.3.3 packageurl-python==0.10.0 packaging==21.3 +packvers==21.5 parameter-expansion-patched==0.3.1 pdfminer.six==20220524 pefile==2022.5.30 -pip-requirements-parser==31.2.0 +pip-requirements-parser==32.0.1 pkginfo2==30.0.0 pluggy==1.0.0 plugincode==31.0.0 diff --git a/setup-mini.cfg b/setup-mini.cfg index 410e7cb6868..3720428bc7c 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -1,6 +1,6 @@ [metadata] name = scancode-toolkit-mini -version = 31.2.1 +version = 31.2.3 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 @@ -59,7 +59,7 @@ zip_safe = false py_modules = scancode_config -python_requires = >=3.7.* +python_requires = >=3.7 install_requires = attrs >= 18.1, !=20.1.0 @@ -71,11 +71,11 @@ install_requires = commoncode >= 31.0.0 container-inspector >= 31.0.0 debian-inspector >= 31.0.0 - dparse2 >= 0.6.1 + dparse2 >= 0.7.0 fasteners fingerprints >= 0.6.0 ftfy >= 6.0.0 - gemfileparser >= 0.7.0 + gemfileparser2 >= 0.9.0 html5lib importlib_metadata intbitset >= 3.0.0 @@ -87,13 +87,13 @@ install_requires = lxml >= 4.9.1 MarkupSafe >= 1.0 packageurl_python >= 0.9.0 - packaging >= 21.0.0 + packvers >= 21.0.0 # use temp advanced patched release parameter-expansion-patched >= 0.3.1 pdfminer.six >= 20200101 pefile >= 2020.1.1 pkginfo2 >= 30.0.0 - pip-requirements-parser >= 31.2.0 + pip-requirements-parser >= 32.0.1 pluggy >= 1.0.0 plugincode >= 31.0.0 publicsuffix2 @@ -103,7 +103,7 @@ install_requires = pymaven_patch >= 0.2.8 requests >= 2.7.0 saneyaml >= 0.5.2 - spdx_tools >= 0.7.0a3 + spdx_tools == 0.7.0a3 text_unidecode >= 1.0 toml >= 0.10.0 urlpy diff --git a/setup.cfg b/setup.cfg index 057c3da657b..ee7eea937b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scancode-toolkit -version = 31.2.1 +version = 31.2.3 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 @@ -59,7 +59,7 @@ zip_safe = false py_modules = scancode_config -python_requires = >=3.7.* +python_requires = >=3.7 install_requires = attrs >= 18.1, !=20.1.0 @@ -71,7 +71,7 @@ install_requires = commoncode >= 31.0.0 container-inspector >= 31.0.0 debian-inspector >= 31.0.0 - dparse2 >= 0.6.1 + dparse2 >= 0.7.0 fasteners fingerprints >= 0.6.0 ftfy >= 6.0.0 @@ -87,13 +87,13 @@ install_requires = lxml >= 4.9.1 MarkupSafe >= 1.0 packageurl_python >= 0.9.0 - packaging >= 21.0.0 + packvers >= 21.0.0 # use temp advanced patched release parameter-expansion-patched >= 0.3.1 pdfminer.six >= 20200101 pefile >= 2020.1.1 pkginfo2 >= 30.0.0 - pip-requirements-parser >= 31.2.0 + pip-requirements-parser >= 32.0.1 pluggy >= 1.0.0 plugincode >= 31.0.0 publicsuffix2 @@ -103,7 +103,7 @@ install_requires = pymaven_patch >= 0.2.8 requests >= 2.7.0 saneyaml >= 0.5.2 - spdx_tools >= 0.7.0a3 + spdx_tools == 0.7.0a3 text_unidecode >= 1.0 toml >= 0.10.0 urlpy diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index e64cd490520..d4c0e3a8e06 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -24,17 +24,17 @@ import dparse2 # NOTE: we always want to use the external library rather than the built-in for now import importlib_metadata -import packaging +import packvers as packaging import pip_requirements_parser import pkginfo2 from commoncode import fileutils from commoncode.fileutils import as_posixpath from commoncode.resource import Resource -from packaging.specifiers import SpecifierSet from packageurl import PackageURL -from packaging import markers -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name +from packvers.specifiers import SpecifierSet +from packvers import markers +from packvers.requirements import Requirement +from packvers.utils import canonicalize_name from packagedcode import models from packagedcode.utils import build_description diff --git a/src/scancode/outdated.py b/src/scancode/outdated.py index 5e03195d11f..d21345cf128 100644 --- a/src/scancode/outdated.py +++ b/src/scancode/outdated.py @@ -33,7 +33,7 @@ import logging from os import path -from packaging import version as packaging_version +from packvers import version as packaging_version import requests from requests.exceptions import ConnectionError @@ -114,7 +114,6 @@ def build_outdated_message(installed_version, release_date, newer_version=''): 'Visit https://github.com/nexB/scancode-toolkit/releases for details.' ) return msg - def check_scancode_version( @@ -155,8 +154,8 @@ def fetch_newer_version( State is stored in the scancode_cache_dir. If `force` is True, redo a PyPI remote check. """ - installed_version = packaging_version.parse(installed_version) try: + installed_version = packaging_version.parse(installed_version) state = VersionCheckState() current_time = datetime.datetime.utcnow() @@ -186,16 +185,6 @@ def fetch_newer_version( latest_version = packaging_version.parse(latest_version) - # Our git version string is not PEP 440 compliant, and thus improperly - # parsed via most 3rd party version parsers. We handle this case by - # pulling out the "base" release version by split()-ting on "post". - # - # For example, "3.1.2.post351.850399ba3" becomes "3.1.2" - if isinstance(installed_version, packaging_version.LegacyVersion): - installed_version = installed_version.split('post') - installed_version = installed_version[0] - installed_version = packaging_version.parse(installed_version) - # Determine if our latest_version is older if (installed_version < latest_version and installed_version.base_version != latest_version.base_version): diff --git a/src/scancode_config.py b/src/scancode_config.py index 30966cd1db1..a2351baca80 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -77,7 +77,7 @@ def _create_dir(location): # in case package is not installed or we do not have setutools/pkg_resources # on hand fall back to this version -__version__ = '31.2.1' +__version__ = '31.2.3' # used to warn user when the version is out of date __release_date__ = datetime.datetime(2022, 10, 5) @@ -87,7 +87,7 @@ def _create_dir(location): __output_format_version__ = '2.0.0' # -spdx_license_list_version = '3.17' +spdx_license_list_version = '3.18' try: from pkg_resources import get_distribution, DistributionNotFound diff --git a/tests/licensedcode/test_zzzz_cache.py b/tests/licensedcode/test_zzzz_cache.py index ad9740f75e4..d9c89710551 100644 --- a/tests/licensedcode/test_zzzz_cache.py +++ b/tests/licensedcode/test_zzzz_cache.py @@ -135,7 +135,9 @@ def test_get_spdx_symbols_fails_on_duplicates(self): cache.get_spdx_symbols(licenses_db=test_licenses) self.fail('ValueError not raised!') except ValueError as e: - assert 'Duplicated SPDX license key' in str(e) + msg = str(e) + assert msg.startswith('Duplicated') + assert 'SPDX license key' in msg def test_get_spdx_symbols_fails_on_duplicated_other_spdx_keys(self): test_dir = self.get_test_loc('spdx/db-dupe-other') @@ -145,7 +147,9 @@ def test_get_spdx_symbols_fails_on_duplicated_other_spdx_keys(self): cache.get_spdx_symbols(licenses_db=test_licenses) self.fail('ValueError not raised!') except ValueError as e: - assert 'Duplicated "other" SPDX license key' in str(e) + msg = str(e) + assert msg.startswith('Duplicated') + assert 'SPDX license key' in msg def test_get_spdx_symbols_checks_duplicates_with_deprecated_on_live_db(self): from licensedcode.models import load_licenses diff --git a/tests/packagedcode/data/pypi/requirements_txt/invalid_spec/output.expected.json b/tests/packagedcode/data/pypi/requirements_txt/invalid_spec/output.expected.json index 9daeb298df0..4634134661b 100644 --- a/tests/packagedcode/data/pypi/requirements_txt/invalid_spec/output.expected.json +++ b/tests/packagedcode/data/pypi/requirements_txt/invalid_spec/output.expected.json @@ -28,29 +28,7 @@ "source_packages": [], "file_references": [], "extra_data": {}, - "dependencies": [ - { - "purl": "pkg:pypi/test", - "extracted_requirement": "test>>1.2.0", - "scope": "install", - "is_runtime": true, - "is_optional": false, - "is_resolved": false, - "resolved_package": {}, - "extra_data": { - "is_editable": false, - "link": null, - "hash_options": [], - "is_constraint": false, - "is_archive": null, - "is_wheel": false, - "is_url": null, - "is_vcs_url": null, - "is_name_at_url": false, - "is_local_path": null - } - } - ], + "dependencies": [], "repository_homepage_url": null, "repository_download_url": null, "api_data_url": null, diff --git a/tests/scancode/test_outdated.py b/tests/scancode/test_outdated.py index 0c328707ec9..cdac7853b2b 100644 --- a/tests/scancode/test_outdated.py +++ b/tests/scancode/test_outdated.py @@ -182,10 +182,10 @@ def jget(*args, **kwargs): installed_version='3.1.2.post351.850399bc3', force=True, ) - assert not result + assert result == '3.1.2' result = outdated.check_scancode_version( installed_version='3.1.2.post351.850399bc3', force=True, ) - assert not result + assert result