Introduced pyproject.toml and moved static metadata from setup.py (#1592

) Introduced pyproject.toml and moved static metadata from setup.py
confluentinc · Oct 18, 2024 · 42f4487 · 42f4487
1 parent e17f6e2
commit 42f4487
Show file tree

Hide file tree

Showing 37 changed files with 149 additions and 169 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 v2.6.1 is a maintenance release with the following fixes and enhancements:
 
+ - Migrated build system from `setup.py` to `pyproject.toml` in accordance with `PEP 517` and `PEP 518`, improving project configuration, build system requirements management, and compatibility with modern Python packaging tools like `pip` and `build`.
  - Added an example for OAUTH OIDC producer with support for confluent cloud (#1769, @sarwarbhuiyan)
 
 confluent-kafka-python is based on librdkafka v2.6.1, see the

diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -5,19 +5,19 @@ This document provides information useful to developers working on confluent-kaf
 
 ## Build
 
-    $ python setup.py build
+    $ python -m build
 
 If librdkafka is installed in a non-standard location provide the include and library directories with:
 
-    $ C_INCLUDE_PATH=/path/to/include LIBRARY_PATH=/path/to/lib python setup.py ...
+    $ C_INCLUDE_PATH=/path/to/include LIBRARY_PATH=/path/to/lib python -m build
 
 **Note**: On Windows the variables for Visual Studio are named INCLUDE and LIB 
 
 ## Generate Documentation
 
-Install sphinx and sphinx_rtd_theme packages:
+Install docs dependencies:
 
-    $ pip install sphinx sphinx_rtd_theme
+    $ pip install .[docs]
 
 Build HTML docs:
 

diff --git a/LICENSE.txt → LICENSE b/LICENSE.txt → LICENSE
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,4 +1,4 @@
 include README.md
-include LICENSE.txt
-include test-requirements.txt
 include src/confluent_kafka/src/*.[ch]
+prune tests
+prune docs
diff --git a/Makefile b/Makefile
@@ -5,7 +5,8 @@ all:
 
 
 clean:
-	python setup.py clean
+	pip cache purge
+	rm -rf dist
 	make -C docs clean
 
 .PHONY: docs

diff --git a/docs/conf.py b/docs/conf.py
@@ -13,14 +13,6 @@
 # serve to show the default.
 
 import sphinx_rtd_theme
-import sys
-import os
-from glob import glob
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path[:0] = [os.path.abspath(x) for x in glob('../build/lib.*')]
 
 # -- General configuration ------------------------------------------------
 

diff --git a/examples/README.md b/examples/README.md
@@ -30,8 +30,8 @@ To setup a venv with the latest release version of confluent-kafka and dependenc
 ```
 $ python3 -m venv venv_examples
 $ source venv_examples/bin/activate
-$ cd examples
-$ pip install -r requirements.txt
+$ pip install confluent_kafka
+$ pip install -r requirements/requirements-examples.txt
 ```
 
 To setup a venv that uses the current source tree version of confluent_kafka, you
@@ -42,9 +42,7 @@ need to have a C compiler and librdkafka installed
 ```
 $ python3 -m venv venv_examples
 $ source venv_examples/bin/activate
-$ python setup.py develop
-$ cd examples
-$ pip install -r requirements.txt
+$ pip install .[examples]
 ```
 
 When you're finished with the venv:

diff --git a/examples/docker/Dockerfile.alpine b/examples/docker/Dockerfile.alpine
@@ -73,9 +73,10 @@ RUN \
     mkdir -p /usr/src/confluent-kafka-python && \
     cd /usr/src/confluent-kafka-python && \
     rm -rf build && \
-    python3 setup.py clean -a && \
-    python3 setup.py build && \
-    python3 setup.py install && \
+    rm -rf dist && \
+    python3 -m pip install build \
+    python3 -m build && \
+    python3 -m pip install dist/confluent_kafka*whl && \
     cd / && \
     rm -rf /usr/src/confluent-kafka-python
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,67 @@
+[build-system]
+requires = [ "setuptools>=62", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "confluent-kafka"
+version = "2.6.0.post1.dev1"
+description = "Confluent's Python client for Apache Kafka"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Topic :: Software Development :: Libraries :: Python Modules"]
+readme = "README.md"
+license = { file = "LICENSE" }
+requires-python = ">=3.7"
+dynamic = ["dependencies", "optional-dependencies"]
+
+[[project.authors]]
+name = "Confluent Inc."
+email = "support@confluent.io"
+
+[project.urls]
+Homepage = "https://github.com/confluentinc/confluent-kafka-python"
+
+[tool.setuptools]
+include-package-data = false
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements/requirements.txt"]}
+optional-dependencies.schemaregistry = { file = ["requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.avro = { file = ["requirements/requirements-avro.txt", "requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.json = { file = ["requirements/requirements-json.txt", "requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.protobuf = { file = ["requirements/requirements-protobuf.txt", "requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.dev = { file = [
+    "requirements/requirements-docs.txt",
+    "requirements/requirements-examples.txt",
+    "requirements/requirements-tests.txt",
+    "requirements/requirements-schemaregistry.txt",
+    "requirements/requirements-avro.txt",
+    "requirements/requirements-json.txt",
+    "requirements/requirements-protobuf.txt"] }
+optional-dependencies.docs = { file = [
+    "requirements/requirements-docs.txt",
+    "requirements/requirements-schemaregistry.txt",
+    "requirements/requirements-avro.txt",
+    "requirements/requirements-json.txt",
+    "requirements/requirements-protobuf.txt"] }
+optional-dependencies.tests = { file = [
+    "requirements/requirements-tests.txt",
+    "requirements/requirements-schemaregistry.txt",
+    "requirements/requirements-avro.txt",
+    "requirements/requirements-json.txt",
+    "requirements/requirements-protobuf.txt"] }
+optional-dependencies.examples = { file = ["requirements/requirements-examples.txt"] }
+optional-dependencies.soaktest = { file = ["requirements/requirements-soaktest.txt"] }
+optional-dependencies.all = { file = [
+    "requirements/requirements-soaktest.txt",
+    "requirements/requirements-docs.txt",
+    "requirements/requirements-examples.txt",
+    "requirements/requirements-tests.txt",
+    "requirements/requirements-schemaregistry.txt",
+    "requirements/requirements-avro.txt",
+    "requirements/requirements-json.txt",
+    "requirements/requirements-protobuf.txt"] }
diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt
@@ -0,0 +1,9 @@
+-r requirements.txt
+-r requirements-schemaregistry.txt
+-r requirements-avro.txt
+-r requirements-protobuf.txt
+-r requirements-json.txt
+-r requirements-examples.txt
+-r requirements-tests.txt
+-r requirements-docs.txt
+-r requirements-soaktest.txt
diff --git a/requirements/requirements-avro.txt b/requirements/requirements-avro.txt
@@ -0,0 +1,4 @@
+fastavro < 1.8.0; python_version == "3.7"
+fastavro < 2; python_version > "3.7"
+requests
+avro>=1.11.1,<2
diff --git a/docs/requirements.txt → requirements/requirements-docs.txt b/docs/requirements.txt → requirements/requirements-docs.txt
diff --git a/examples/requirements.txt → requirements/requirements-examples.txt b/examples/requirements.txt → requirements/requirements-examples.txt
diff --git a/requirements/requirements-json.txt b/requirements/requirements-json.txt
@@ -0,0 +1,2 @@
+pyrsistent
+jsonschema
diff --git a/requirements/requirements-protobuf.txt b/requirements/requirements-protobuf.txt
@@ -0,0 +1 @@
+protobuf
diff --git a/requirements/requirements-schemaregistry.txt b/requirements/requirements-schemaregistry.txt
@@ -0,0 +1 @@
+requests
diff --git a/tests/soak/requirements.txt → requirements/requirements-soaktest.txt b/tests/soak/requirements.txt → requirements/requirements-soaktest.txt
diff --git a/requirements/requirements-tests-install.txt b/requirements/requirements-tests-install.txt
@@ -0,0 +1,5 @@
+-r requirements-tests.txt  
+-r requirements-schemaregistry.txt  
+-r requirements-avro.txt  
+-r requirements-protobuf.txt  
+-r requirements-json.txt 
diff --git a/requirements/requirements-tests.txt b/requirements/requirements-tests.txt
@@ -0,0 +1,8 @@
+# core test requirements
+urllib3<2.0.0;python_version<="3.7"
+urllib3 >= 2.0.0,<3; python_version > "3.7"
+flake8
+pytest
+pytest-timeout
+requests-mock
+trivup>=0.8.3
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
diff --git a/setup.py b/setup.py
@@ -1,41 +1,14 @@
 #!/usr/bin/env python
 
 import os
-from setuptools import setup, find_packages
+from setuptools import setup
 from distutils.core import Extension
 import platform
 
 work_dir = os.path.dirname(os.path.realpath(__file__))
 mod_dir = os.path.join(work_dir, 'src', 'confluent_kafka')
 ext_dir = os.path.join(mod_dir, 'src')
 
-INSTALL_REQUIRES = [
-    'futures;python_version<"3.2"',
-    'enum34;python_version<"3.4"',
-]
-
-TEST_REQUIRES = [
-    'pytest==4.6.4;python_version<"3.0"',
-    'pytest;python_version>="3.0"',
-    'pytest-timeout',
-    'flake8'
-]
-
-DOC_REQUIRES = ['sphinx', 'sphinx-rtd-theme']
-
-SCHEMA_REGISTRY_REQUIRES = ['requests']
-
-AVRO_REQUIRES = ['fastavro>=0.23.0,<1.0;python_version<"3.0"',
-                 'fastavro>=1.0;python_version>"3.0"',
-                 'avro>=1.11.1,<2',
-                 ] + SCHEMA_REGISTRY_REQUIRES
-
-JSON_REQUIRES = ['pyrsistent==0.16.1;python_version<"3.0"',
-                 'pyrsistent;python_version>"3.0"',
-                 'jsonschema'] + SCHEMA_REGISTRY_REQUIRES
-
-PROTO_REQUIRES = ['protobuf'] + SCHEMA_REGISTRY_REQUIRES
-
 # On Un*x the library is linked as -lrdkafka,
 # while on windows we need the full librdkafka name.
 if platform.system() == 'Windows':
@@ -52,45 +25,4 @@
                             os.path.join(ext_dir, 'AdminTypes.c'),
                             os.path.join(ext_dir, 'Admin.c')])
 
-
-def get_install_requirements(path):
-    content = open(os.path.join(os.path.dirname(__file__), path)).read()
-    return [
-        req
-        for req in content.split("\n")
-        if req != '' and not req.startswith('#')
-    ]
-
-
-trove_classifiers = [
-    'Development Status :: 5 - Production/Stable',
-    'Intended Audience :: Developers',
-    'License :: OSI Approved :: Apache Software License',
-    'Programming Language :: Python',
-    'Programming Language :: Python :: 2.7',
-    'Programming Language :: Python :: 3',
-    'Topic :: Software Development :: Libraries :: Python Modules',
-]
-
-setup(name='confluent-kafka',
-      # Make sure to bump CFL_VERSION* in confluent_kafka/src/confluent_kafka.h
-      # and version in docs/conf.py.
-      version='2.6.0',
-      description='Confluent\'s Python client for Apache Kafka',
-      author='Confluent Inc',
-      author_email='support@confluent.io',
-      url='https://github.com/confluentinc/confluent-kafka-python',
-      ext_modules=[module],
-      packages=find_packages('src'),
-      package_dir={'': 'src'},
-      data_files=[('', [os.path.join(work_dir, 'LICENSE.txt')])],
-      install_requires=INSTALL_REQUIRES,
-      classifiers=trove_classifiers,
-      extras_require={
-          'schema-registry': SCHEMA_REGISTRY_REQUIRES,
-          'avro': AVRO_REQUIRES,
-          'json': JSON_REQUIRES,
-          'protobuf': PROTO_REQUIRES,
-          'dev': TEST_REQUIRES + AVRO_REQUIRES,
-          'doc': DOC_REQUIRES + AVRO_REQUIRES
-      })
+setup(ext_modules=[module])
diff --git a/src/confluent_kafka/avro/requirements.txt b/src/confluent_kafka/avro/requirements.txt
diff --git a/src/confluent_kafka/requirements.txt b/src/confluent_kafka/requirements.txt
diff --git a/src/confluent_kafka/schema_registry/requirements.txt b/src/confluent_kafka/schema_registry/requirements.txt
diff --git a/src/confluent_kafka/src/confluent_kafka.h b/src/confluent_kafka/src/confluent_kafka.h
@@ -36,7 +36,7 @@
 
 
 /**
- * @brief confluent-kafka-python version, must match that of setup.py.
+ * @brief confluent-kafka-python version, must match that of pyproject.toml.
  *
  * Hex version representation:
  *  0xMMmmRRPP

diff --git a/tests/README.md b/tests/README.md
@@ -18,9 +18,7 @@ A python3 env suitable for running tests:
 
     $ python3 -m venv venv_test
     $ source venv_test/bin/activate
-    $ pip install -r tests/requirements.txt
-    $ python setup.py build
-    $ python setup.py install
+    $ python3 -m pip install .[tests]
 
 When you're finished with it:
 

diff --git a/tests/requirements.txt b/tests/requirements.txt
diff --git a/tests/soak/bootstrap.sh b/tests/soak/bootstrap.sh
@@ -45,9 +45,7 @@ source $venv/bin/activate
 
 pip install -U pip
 
-pip install -v .
-
-pip install -r tests/soak/requirements.txt
+pip install -v .[soaktest]
 
 popd # ..python
 

diff --git a/tests/soak/build.sh b/tests/soak/build.sh
@@ -32,8 +32,6 @@ set -u
 pushd confluent-kafka-python
 git fetch --tags
 git checkout $cflpy_version
-python3 setup.py clean -a
-python3 setup.py build
 python3 -m pip install .
 popd