From 9e113d613577aa4f9cabc94bff00f53bb7daa94e Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 21:06:15 +0100 Subject: [PATCH 01/13] docs/RELEASE: use build for building dists This is the PyPA recommend tool for building Python packages: https://packaging.python.org/guides/distributing-packages-using-setuptools/ Furthermore, direct invocation of setup.py is long ago deprecated: https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html Signed-off-by: Joshua Lock --- docs/RELEASE.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/RELEASE.md b/docs/RELEASE.md index b69ae1cdf2..f67cef3290 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -9,8 +9,7 @@ * Test packaging, uploading to Test PyPI and installing from a virtual environment (ensure commands invoking `python` below are using Python 3) * Remove existing dist build dirs - * Create source dist `python3 setup.py sdist` - * Create wheel `python3 setup.py bdist_wheel` + * Create source dist and wheel `python3 -m build` * Sign the dists `gpg --detach-sign -a dist/tuf-vA.B.C.tar.gz` * Upload to test PyPI `twine upload --repository testpypi dist/*` * Verify the uploaded package https://testpypi.python.org/pypi/tuf/ @@ -24,13 +23,12 @@ * Create a package for the formal release (ensure commands invoking `python` below are using Python 3) * Remove existing dist build dirs - * Create source dist `python3 setup.py sdist` - * Create wheel `python3 setup.py bdist_wheel` + * Create source dist and wheel `python3 -m build` * Sign source dist `gpg --detach-sign -a dist/tuf-vA.B.C.tar.gz` * Sign wheel `gpg --detach-sign -a dist/tuf-vA.B.C-py3-none-any.whl` * Upload to test PyPI `twine upload --repository testpypi dist/*` * Verify the uploaded package https://testpypi.python.org/pypi/tuf/ * Upload to PyPI `twine upload dist/*` -* Attach the signed dists to the release on GitHub +* Attach both signed dists and their detached signatures to the release on GitHub * Announce the release on [#tuf on CNCF Slack](https://cloud-native.slack.com/archives/C8NMD3QJ3) * Ensure [POUF 1](https://github.com/theupdateframework/taps/blob/master/POUFs/reference-POUF/pouf1.md), for the reference implementation, is up-to-date From 2ed51167f62edd01b104640a8f2f23195a677e24 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 21:10:55 +0100 Subject: [PATCH 02/13] build: add build and release deps to requirements-dev build, twine and wheel packages should all be installed in order to be able to build and release python-tuf -- add those dependencies to requirements-dev.txt Signed-off-by: Joshua Lock --- requirements-dev.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index f8748752a5..2afa895fb3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,8 @@ # Install tuf in editable mode and requirements for local testing with tox, # and also for running test suite or individual tests manually +build tox +twine +wheel -r requirements-test.txt -e . From 2ab518b3f082535c603b567b4e3a5413ebe45de0 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 21:12:16 +0100 Subject: [PATCH 03/13] build: cleanup setup.cfg * List license files in a new metadata section * Remove .travis.yml from check-manifest section's ignore entry Signed-off-by: Joshua Lock --- setup.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 366a2fbb53..d7525714e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,9 @@ +[metadata] +license_files = LICENSE LICENSE-MIT + [check-manifest] ignore = requirements-dev.txt - .travis.yml .coveragerc [mypy] From 08decea2d004ff091d99a076409aebb0ff532141 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 21:37:34 +0100 Subject: [PATCH 04/13] Remove unused .gitmodules directory Signed-off-by: Joshua Lock --- .gitmodules | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29bb2..0000000000 From 4fd35434bd211e99200aa6904fe95d9038174640 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 21:37:50 +0100 Subject: [PATCH 05/13] build: update MANIFEST.in to match sdist Update the MANIFEST.in to be explicit about what we choose to ship in our sdist. This _does not_ result in any additional files being included in our sdist, but does remove warnings from build. Signed-off-by: Joshua Lock --- MANIFEST.in | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6e2a7cbb01..764e1f58a6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,10 @@ +# License files include LICENSE* -include README.md + +# Test related files include tox.ini +recursive-include tests *.py +recursive-include tests *.md include tests/repository_data/keystore/delegation_key include tests/repository_data/keystore/root_key* include tests/repository_data/keystore/snapshot_key @@ -8,22 +12,24 @@ include tests/repository_data/keystore/targets_key include tests/repository_data/keystore/timestamp_key include tests/ssl_certs/*.crt include tests/ssl_certs/*.key - -recursive-include docs *.txt -recursive-include docs *.md -recursive-include docs *.rst -recursive-include docs/images *.png -recursive-include tuf/scripts *.py -recursive-include examples * -recursive-include tests *.py -recursive-include tests *.pem recursive-include tests *.json recursive-include tests *.txt recursive-include tests *.cfg recursive-include tests *.coveragerc -recursive-include tests *.gz recursive-include tests *.pub -recursive-include tuf *.gitignore + +# Development related files +include requirements*.txt +include pylintrc +include tuf/api/pylintrc + +# Docs +include docs/conf.py recursive-include tuf *.md -recursive-include tuf *.rst -recursive-include tuf *.yml +recursive-include docs *.txt +recursive-include docs *.md +recursive-include docs *.rst + +# To remove +recursive-include tuf/scripts *.py + From 36242adc748c11e71adff8395c0a1678296bb373 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 21:40:07 +0100 Subject: [PATCH 06/13] build: recommend using build, not setup.py Invoking setup.py directly is deprecated, see: https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html Therefore: * remove the executable bit from setup.py's permissions * remove the shebang entry from setup.py * update the comments in setup.py to recommend using build to create dists and pip to install them Signed-off-by: Joshua Lock --- setup.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 0716124e07..b5c39baf4d --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Copyright 2013 - 2018, New York University and the TUF contributors # SPDX-License-Identifier: MIT OR Apache-2.0 @@ -23,7 +21,7 @@ distributed to other users. The packaged source is saved to the 'dist' folder in the current directory. - $ python3 setup.py sdist + $ python3 -m build --sdist INSTALLATION OPTIONS @@ -51,16 +49,12 @@ run one of the following shell commands: Install to the global site-packages directory. - $ python3 setup.py install + $ python3 -m pip install Install to the user site-packages directory. - $ python3 setup.py install --user - - Install to a chosen directory. - $ python3 setup.py install --home= - + $ python3 -m pip install --user - Note: The last two installation options may require modification of + Note: The last installation option may require modification of Python's search path (i.e., 'sys.path') or updating an OS environment variable. For example, installing to the user site-packages directory might result in the installation of TUF scripts to '~/.local/bin'. The user may From d3e34acd9887e024c7c1eff593828f52b12d68dd Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 21:42:01 +0100 Subject: [PATCH 07/13] build: add docs to project_urls Add a 'Documentation' entry to project_urls pointing to our stable docs on readthedocs.io. This will result in a 'Documentation' entry under the 'Project links' section on PyPI. Signed-off-by: Joshua Lock --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b5c39baf4d..9cbf6ce464 100644 --- a/setup.py +++ b/setup.py @@ -101,8 +101,9 @@ 'Topic :: Software Development' ], project_urls={ - 'Source': 'https://github.com/theupdateframework/python-tuf', - 'Issues': 'https://github.com/theupdateframework/python-tuf/issues' + 'Documentation': 'https://theupdateframework.readthedocs.io/en/stable/', + 'Issues': 'https://github.com/theupdateframework/python-tuf/issues', + 'Source': 'https://github.com/theupdateframework/python-tuf' }, python_requires="~=3.6", install_requires = [ From 38ea974674af754ee73664ce58dec8cffeda3ad0 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 22:11:02 +0100 Subject: [PATCH 08/13] build: convert to static setuptools metadata This is preferred and removes the crutch of invoking setup.py as a CLI tool for building dists. Signed-off-by: Joshua Lock --- setup.cfg | 44 ++++++++++++++++++++ setup.py | 118 ------------------------------------------------------ 2 files changed, 44 insertions(+), 118 deletions(-) delete mode 100644 setup.py diff --git a/setup.cfg b/setup.cfg index d7525714e3..c9f3a64e53 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,50 @@ [metadata] +name = tuf +version = 0.19.0 +author = https://www.updateframework.com +author_email = theupdateframework@googlegroups.com +description = A secure updater framework for Python +long_description = file: README.md +long_description_content_type = text/markdown +url = https://www.updateframework.com +project_urls = + Documentation = https://theupdateframework.readthedocs.io/en/stable/ + Issues = https://github.com/theupdateframework/python-tuf/issues + Source = https://github.com/theupdateframework/python-tuf +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: MIT License + License :: OSI Approved :: Apache Software License + Operating System :: POSIX + Operating System :: POSIX :: Linux + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft :: Windows + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + Topic :: Security + Topic :: Software Development +keywords = update updater secure authentication key compromise revocation license_files = LICENSE LICENSE-MIT +[options] +packages = find: +scripts = + tuf/scripts/repo.py + tuf/scripts/client.py +python_requires = ~=3.6 +install_requires = + requests>=2.19.1 + securesystemslib>=0.20.0 + +[options.packages.find] +exclude = tests + [check-manifest] ignore = requirements-dev.txt diff --git a/setup.py b/setup.py deleted file mode 100644 index 9cbf6ce464..0000000000 --- a/setup.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2013 - 2018, New York University and the TUF contributors -# SPDX-License-Identifier: MIT OR Apache-2.0 - -""" - - setup.py - - - Vladimir Diaz - - - March 2013. - - - See LICENSE-MIT OR LICENSE for licensing information. - - - BUILD SOURCE DISTRIBUTION - - The following shell command generates a TUF source archive that can be - distributed to other users. The packaged source is saved to the 'dist' - folder in the current directory. - - $ python3 -m build --sdist - - - INSTALLATION OPTIONS - - pip - installing and managing Python packages (recommended): - - # Installing from Python Package Index (https://pypi.python.org/pypi). - $ python3 -m pip install tuf - - # Installing from local source archive. - $ python3 -m pip install - - # Or from the root directory of the unpacked archive. - $ python3 -m pip install . - - # Installing optional requirements (i.e., after installing tuf). - # Support for creation of Ed25519 signatures and support for RSA and ECDSA - # signatures in general requires optional dependencies: - $ python3 -m pip install securesystemslib[crypto,pynacl] - - - Alternate installation options: - - Navigate to the root directory of the unpacked archive and - run one of the following shell commands: - - Install to the global site-packages directory. - $ python3 -m pip install - - Install to the user site-packages directory. - $ python3 -m pip install --user - - Note: The last installation option may require modification of - Python's search path (i.e., 'sys.path') or updating an OS environment - variable. For example, installing to the user site-packages directory might - result in the installation of TUF scripts to '~/.local/bin'. The user may - then be required to update his $PATH variable: - $ export PATH=$PATH:~/.local/bin -""" - -from setuptools import setup -from setuptools import find_packages - - -with open('README.md') as file_object: - long_description = file_object.read() - - -setup( - name = 'tuf', - version = '0.19.0', # If updating version, also update it in tuf/__init__.py - description = 'A secure updater framework for Python', - long_description = long_description, - long_description_content_type='text/markdown', - author = 'https://www.updateframework.com', - author_email = 'theupdateframework@googlegroups.com', - url = 'https://www.updateframework.com', - keywords = 'update updater secure authentication key compromise revocation', - classifiers = [ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'License :: OSI Approved :: Apache Software License', - 'Natural Language :: English', - 'Operating System :: POSIX', - 'Operating System :: POSIX :: Linux', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: Implementation :: CPython', - 'Topic :: Security', - 'Topic :: Software Development' - ], - project_urls={ - 'Documentation': 'https://theupdateframework.readthedocs.io/en/stable/', - 'Issues': 'https://github.com/theupdateframework/python-tuf/issues', - 'Source': 'https://github.com/theupdateframework/python-tuf' - }, - python_requires="~=3.6", - install_requires = [ - 'requests>=2.19.1', - 'securesystemslib>=0.20.0' - ], - packages = find_packages(exclude=['tests']), - scripts = [ - 'tuf/scripts/repo.py', - 'tuf/scripts/client.py' - ] -) From ac0ea24ca96ccfb2cbf30c40157eec0b0dd20636 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Thu, 21 Oct 2021 10:53:35 +0100 Subject: [PATCH 09/13] Remove references to setup.py We now use a static setup.cfg instead, update __init__.py and docs/RELEASE.md to point there. Signed-off-by: Joshua Lock --- docs/RELEASE.md | 2 +- tuf/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/RELEASE.md b/docs/RELEASE.md index f67cef3290..c964407aed 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -5,7 +5,7 @@ tracked by git * Ensure `docs/CHANGELOG.md` contains a one-line summary of each [notable change](https://keepachangelog.com/) since the prior release -* Update `setup.py` and `tuf/__init__.py` to the new version number vA.B.C +* Update `setup.cfg` and `tuf/__init__.py` to the new version number vA.B.C * Test packaging, uploading to Test PyPI and installing from a virtual environment (ensure commands invoking `python` below are using Python 3) * Remove existing dist build dirs diff --git a/tuf/__init__.py b/tuf/__init__.py index e56bdf943b..cca809ba0d 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -1,5 +1,5 @@ # This value is used in the requests user agent. -# setup.py has it hard-coded separately. +# setup.cfg has it hard-coded separately. # Currently, when the version is changed, it must be set in both locations. # TODO: Single-source the version number. __version__ = "0.19.0" From 52ad17a710375291ba598a5087ceda9459977891 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 22:11:59 +0100 Subject: [PATCH 10/13] build: update check-manifest options Signed-off-by: Joshua Lock --- setup.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index c9f3a64e53..099d3fa94d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,8 +47,10 @@ exclude = tests [check-manifest] ignore = - requirements-dev.txt - .coveragerc + .fossa.yml + .readthedocs.yaml + docs/images/* + docs/latex/* [mypy] warn_unused_configs = True From ada35c9e8fcc36874bff21ed931f023399db5d38 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 22:16:01 +0100 Subject: [PATCH 11/13] build: add pyproject.toml to list build tools Per PEP 517 the pyproject.toml file lists tools required when building the project with build: https://pypa-build.readthedocs.io/en/stable/index.html Signed-off-by: Joshua Lock --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..2f21011953 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=40.8.0", "wheel"] +build-backend = "setuptools.build_meta" From 352f6c2c56c835e78b5869fc1c6bf22b39ca2a39 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 19 Oct 2021 22:20:21 +0100 Subject: [PATCH 12/13] Add vscode project directory to gitignore Signed-off-by: Joshua Lock --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e988195f05..6ca43668d8 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ tests/htmlcov/* *~ *.tmp .pre-commit-config.yaml +.vscode # Debian generated files debian/.debhelper/* From de1a3af0198d95d933bd3c49cef790fd71823fdb Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Wed, 20 Oct 2021 10:49:47 +0100 Subject: [PATCH 13/13] build: more intentional about what's included in sdist Our sdist has typically included everything from git apart from the CI related files (.github/*, .fossa.yml, .readthedocs.yaml). Update our MANIFEST.in and the check-manifest section of setup.cfg to be explicit about this. Signed-off-by: Joshua Lock --- MANIFEST.in | 32 ++++++-------------------------- setup.cfg | 2 -- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 764e1f58a6..06f92591ed 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,35 +1,15 @@ # License files include LICENSE* -# Test related files +# Development and testing +include *.txt +global-include pylintrc include tox.ini -recursive-include tests *.py -recursive-include tests *.md -include tests/repository_data/keystore/delegation_key -include tests/repository_data/keystore/root_key* -include tests/repository_data/keystore/snapshot_key -include tests/repository_data/keystore/targets_key -include tests/repository_data/keystore/timestamp_key -include tests/ssl_certs/*.crt -include tests/ssl_certs/*.key -recursive-include tests *.json -recursive-include tests *.txt -recursive-include tests *.cfg -recursive-include tests *.coveragerc -recursive-include tests *.pub +graft tests -# Development related files -include requirements*.txt -include pylintrc -include tuf/api/pylintrc - -# Docs -include docs/conf.py +# Documentation +graft docs recursive-include tuf *.md -recursive-include docs *.txt -recursive-include docs *.md -recursive-include docs *.rst # To remove recursive-include tuf/scripts *.py - diff --git a/setup.cfg b/setup.cfg index 099d3fa94d..f880a3893d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,8 +49,6 @@ exclude = tests ignore = .fossa.yml .readthedocs.yaml - docs/images/* - docs/latex/* [mypy] warn_unused_configs = True