Prepare release 31

CHANGELOG.rst

@@ -4,17 +4,14 @@ Release notes
 Version (next)
 ------------------------------
 
-- Add ``posix_only`` option to ``commoncode.paths.portable_filename`` and
-  ``commoncode.paths.safe_path``. This option prevents
-  ``commoncode.paths.portable_filename`` and ``commoncode.paths.safe_path`` from
-  replacing filenames and punctuation in filenames that are valid on POSIX
-  operating systems, but not Windows.
 
-Version 31.0.0 - (2022-05-16)
+Version 31.0.0 - (2022-08-24)
 ------------------------------
 
 This is a major version with API-breaking changes in the resource module.
 
+- Drop support for Python 3.6
+
 - The Resource has no rid (resource id) and no pid (parent id). Instead
   we now use internally a simpler mapping of {path: Resource} object.
   As a result the iteration on a Codebase is faster but this requires more
@@ -88,6 +85,14 @@ Other changes:
   The returned directory has a trailing path separator unless with_trail is False.
   The default is True and the default behaviour is unchanged.
 
+- Add ``posix_only`` option to ``commoncode.paths.portable_filename`` and
+  ``commoncode.paths.safe_path``. This option prevents
+  ``commoncode.paths.portable_filename`` and ``commoncode.paths.safe_path`` from
+  replacing filenames and punctuation in filenames that are valid on POSIX
+  operating systems, but not Windows.
+
+- Remove unused intbitset dependency.
+
 
 Version 30.2.0 - (2022-05-02)
 ------------------------------

azure-pipelines.yml

@@ -1,3 +1,4 @@
+
 ################################################################################
 # We use Azure to run the full tests suites on multiple Python 3.x
 # on multiple Windows, macOS and Linux versions all on 64 bits
@@ -6,27 +7,39 @@
 
 jobs:
 
+################################################################################
+# These jobs are using VMs and Azure-provided Pythons 3.8
+################################################################################
+
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu18_cpython
           image_name: ubuntu-18.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython
           image_name: ubuntu-20.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: ubuntu22_cpython
+          image_name: ubuntu-22.04
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1015_cpython
           image_name: macos-10.15
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
@@ -38,11 +51,19 @@ jobs:
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos12_cpython
+          image_name: macos-12
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython
           image_name: windows-2019
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
               all: venv\Scripts\pytest -n 2 -vvs
 
@@ -64,36 +85,42 @@ jobs:
       parameters:
           job_name: ubuntu20_test_all_supported_click_versions
           image_name: ubuntu-20.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
-              click_versions: for clk_ver in 8.0.1 7.1.2 7.1.1 7.1 6.7; do pip install click==$clk_ver; venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; done
+              click_versions: |
+                for clk_ver in 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7;
+                  do
+                    pip install click==$clk_ver;
+                    venv/bin/pytest -vvs tests/test_cliutils_progressbar.py;
+                  done
 
 
 ################################################################################
-# Test using a plain pip install to get the latest of all wheels
+# Tests using a plain pip install to get the latest of all wheels
 ################################################################################
 
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython_latest_from_pip
           image_name: ubuntu-20.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: venv/bin/pip install --force-reinstall --upgrade -e . && venv/bin/pytest -n 2 -vvs
+              all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . && venv/bin/pytest -n 2 -vvs
+
 
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython_latest_from_pip
           image_name: windows-2019
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: venv\Scripts\pip install --force-reinstall --upgrade -e . &&  venv\Scripts\pytest -n 2 -vvs
+              all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e . &&  venv\Scripts\pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos1015_cpython_latest_from_pip
-          image_name: macos-10.15
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          job_name: macos11_cpython_latest_from_pip
+          image_name: macos-11
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: venv/bin/pip install --force-reinstall --upgrade -e . && venv/bin/pytest -n 2 -vvs
+              all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . && venv/bin/pytest -n 2 -vvs

configure

@@ -52,11 +52,19 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 
+################################
+# Install with or without and index. With "--no-index" this is using only local wheels
+# This is an offline mode with no index and no network operations
+# NO_INDEX="--no-index "
+NO_INDEX=""
+
+
 ################################
 # Thirdparty package locations and index handling
-# Find packages from the local thirdparty directory
-if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then
-    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty"
+# Find packages from the local thirdparty directory if present
+THIRDPARDIR=$CFG_ROOT_DIR/thirdparty
+if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then
+    PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR"
 fi
 
 
@@ -182,6 +190,7 @@ while getopts :-: optchar; do
     esac
 done
 
+
 PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS"
 
 find_python

etc/scripts/utils_thirdparty.py

@@ -1678,6 +1678,7 @@ def get_package_version(self, name, version=None):
         """
         if not version:
             versions = list(self._get_package_versions_map(name).values())
+            # return the latest version
             return versions and versions[-1]
         else:
             return self._get_package_versions_map(name).get(version)

requirements-dev.txt

@@ -1,24 +1,31 @@
 aboutcode-toolkit==7.0.2
-bleach==4.1.0
+black==22.6.0
+bleach==5.0.1
 build==0.7.0
 commonmark==0.9.1
-docutils==0.18.1
+docutils==0.19
 et-xmlfile==1.1.0
 execnet==1.9.0
 iniconfig==1.1.1
-jeepney==0.7.1
-keyring==23.4.1
-openpyxl==3.0.9
+isort==5.10.1
+jeepney==0.8.0
+keyring==23.7.0
+mypy-extensions==0.4.3
+openpyxl==3.0.10
+pathspec==0.9.0
 pep517==0.12.0
-pkginfo==1.8.2
+pkginfo==1.8.3
+platformdirs==2.5.2
 py==1.11.0
-pytest==7.0.1
+pytest==7.1.2
 pytest-forked==1.4.0
 pytest-xdist==2.5.0
-readme-renderer==34.0
+readme-renderer==35.0
 requests-toolbelt==0.9.1
-rfc3986==1.5.0
-rich==12.3.0
+rfc3986==2.0.0
+rich==12.5.1
 secretstorage==3.3.2
-tomli==1.2.3
-twine==3.8.0
+tomli==2.0.1
+tqdm==4.64.0
+twine==4.0.1
+typing_extensions==4.3.0

requirements.txt

@@ -3,77 +3,74 @@ banal==1.0.6
 beautifulsoup4==4.11.1
 binaryornot==0.4.4
 boolean.py==4.0
-certifi==2021.10.8
-cffi==1.15.0
-chardet==4.0.0
-charset-normalizer==2.0.12
-click==8.0.4
-colorama==0.4.4
+certifi==2022.6.15
+cffi==1.15.1
+chardet==5.0.0
+charset-normalizer==2.1.0
+click==8.1.3
+colorama==0.4.5
 construct==2.10.68
-container-inspector==31.0.0
-cryptography==36.0.2
-debian-inspector==30.0.0
+container-inspector==31.1.0
+cryptography==37.0.4
+debian-inspector==31.0.0b1
 dockerfile-parse==1.2.0
 dparse2==0.6.1
 extractcode==31.0.0
 extractcode-7z==16.5.210531
 extractcode-libarchive==3.5.1.210531
 fasteners==0.17.3
 fingerprints==1.0.3
-ftfy==6.0.3
+ftfy==6.1.1
 future==0.18.2
 gemfileparser==0.8.0
 html5lib==1.1
 idna==3.3
-importlib-metadata==4.8.3
+importlib-metadata==4.12.0
 inflection==0.5.1
 intbitset==3.0.1
 isodate==0.6.1
-jaraco.functools==3.4.0
+jaraco.functools==3.5.1
 javaproperties==0.8.1
-Jinja2==3.0.3
+Jinja2==3.1.2
 jsonstreams==0.6.0
 license-expression==30.0.0
-lxml==4.8.0
-MarkupSafe==2.0.1
+lxml==4.9.1
+MarkupSafe==2.1.1
 more-itertools==8.13.0
 normality==2.3.3
-packagedcode-msitools==0.101.210706
-packageurl-python==0.9.9
+packageurl-python==0.10.0
 packaging==21.3
 parameter-expansion-patched==0.3.1
-patch==1.16
-pdfminer-six==20220506
-pefile==2021.9.3
+pdfminer.six==20220524
+pefile==2022.5.30
 pip-requirements-parser==31.2.0
 pkginfo2==30.0.0
 pluggy==1.0.0
-plugincode==30.0.0
+plugincode==31.0.0b1
 ply==3.11
 publicsuffix2==2.20191221
 pyahocorasick==2.0.0b1
 pycparser==2.21
 pygmars==0.7.0
 Pygments==2.12.0
 pymaven-patch==0.3.0
-pyparsing==3.0.8
+pyparsing==3.0.9
 pytz==2022.1
 PyYAML==6.0
-rdflib==5.0.0
-regipy==2.3.1
-requests==2.27.1
-rpm-inspector-rpm==4.16.1.3.210404
+rdflib==6.2.0
+requests==2.28.1
 saneyaml==0.5.2
 six==1.16.0
-soupsieve==2.3.1
+soupsieve==2.3.2.post1
 spdx-tools==0.7.0a3
 text-unidecode==1.3
 toml==0.10.2
 typecode==30.0.0
 typecode-libmagic==5.39.210531
-urllib3==1.26.9
+typing-extensions==4.3.0
+urllib3==1.26.11
 urlpy==0.5
 wcwidth==0.2.5
 webencodings==0.5.1
-xmltodict==0.12.0
-zipp==3.6.0
+xmltodict==0.13.0
+zipp==3.8.1

setup.cfg

@@ -35,17 +35,15 @@ packages = find:
 include_package_data = true
 zip_safe = false
 
-python_requires = >=3.6
+python_requires = >=3.7
 
 install_requires =
     attrs >= 18.1, !=20.1.0
     Beautifulsoup4 >= 4.0.0
     click >= 6.7, !=7.0
-    intbitset >= 2.3.0
     requests >= 2.7.0
     saneyaml >= 0.5.2
     text_unidecode >= 1.0
-    typing >=3.6, < 3.7; python_version < "3.7"
 
 
 [options.packages.find]

src/commoncode/resource.py

@@ -267,6 +267,10 @@ def __init__(
 
         ``max_depth`` is the maximum depth of subdirectories to descend below and
         including `location`.
+
+        ``paths`` is an optional list of of path strings that extend from the
+        root ``location``. If provided, the codebase will contain only these
+        paths.
         """
         self.max_depth = max_depth