👷 Divide pip constraints into distinct pairs

.github/workflows/ci-cd.yml

@@ -38,6 +38,13 @@ jobs:
       uses: actions/checkout@v3
       with:
         submodules: true
+    - name: >-
+        Verify that `requirements/runtime-deps.in`
+        is in sync with `setup.cfg`
+      run: |
+        set -eEuo pipefail
+        make sync-direct-runtime-deps
+        git diff --exit-code -- requirements/runtime-deps.in
     - name: Setup Python
       uses: actions/setup-python@v4
       with:
@@ -54,10 +61,10 @@ jobs:
         python -m pip install -U pip wheel setuptools build twine
     - name: Install dependencies
       run: |
-        python -m pip install -r requirements/lint.txt -c requirements/constraints.txt
+        python -m pip install -r requirements/lint.in -c requirements/lint.txt
     - name: Install self
       run: |
-        python -m pip install .
+        python -m pip install . -c requirements/runtime-deps.txt
       env:
         AIOHTTP_NO_EXTENSIONS: 1
     - name: Run mypy
@@ -67,14 +74,14 @@ jobs:
       run: |
         # Some extra requirements are needed to ensure all modules
         # can be scanned by slotscheck.
-        pip install -r requirements/base.txt -c requirements/constraints.txt
+        pip install -r requirements/base.in -c requirements/base.txt
         slotscheck -v -m aiohttp
     - name: Install libenchant
       run: |
         sudo apt install libenchant-2-dev
     - name: Install spell checker
       run: |
-        pip install -r requirements/doc-spelling.txt -c requirements/constraints.txt
+        pip install -r requirements/doc-spelling.in -c requirements/doc-spelling.txt
     - name: Run docs spelling
       run: |
         # towncrier --yes  # uncomment me after publishing a release
@@ -203,7 +210,7 @@ jobs:
         python -m pip install -U pip wheel setuptools build twine
     - name: Install dependencies
       run: |
-        python -m pip install -r requirements/test.txt -c requirements/constraints.txt
+        python -m pip install -r requirements/test.in -c requirements/test.txt
     - name: Cythonize
       if: ${{ matrix.no-extensions == '' }}
       run: |
@@ -294,8 +301,9 @@ jobs:
       run: |
         python -m pip install -U pip wheel setuptools build twine
     - name: Install cython
-      run: |
-        python -m pip install -r requirements/cython.txt -c requirements/constraints.txt
+      run: >-
+        python -m
+        pip install -r requirements/cython.in -c requirements/cython.txt
     - name: Restore llhttp generated files
       uses: actions/download-artifact@v3
       with:
@@ -359,8 +367,9 @@ jobs:
       run: |
         python -m pip install -U pip wheel setuptools build twine
     - name: Install cython
-      run: |
-        python -m pip install -r requirements/cython.txt -c requirements/constraints.txt
+      run: >-
+        python -m
+        pip install -r requirements/cython.in -c requirements/cython.txt
     - name: Restore llhttp generated files
       uses: actions/download-artifact@v3
       with:
@@ -396,9 +405,6 @@ jobs:
       uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Update pip, wheel, setuptools, build, twine
-      run: |
-        python -m pip install -U pip wheel setuptools build twine
     - name: Login
       run: |
         echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token

.github/workflows/update-pre-commit.yml

@@ -14,9 +14,8 @@ jobs:
       with:
         python-version: 3.8
     - name: Install dependencies
-      uses: py-actions/py-dependency-install@v4
-      with:
-        path: requirements/lint.txt
+      run: >-
+        pip install -r requirements/lint.in -c requirements/lint.txt
     - name: Run pre-commit autoupdate
       run: pre-commit autoupdate
     - id: generate_token

.pip-tools.toml

@@ -0,0 +1,4 @@
+[pip-tools]
+allow-unsafe = true
+resolver = "backtracking"
+strip-extras = true

.pre-commit-config.yaml

@@ -46,7 +46,7 @@ repos:
       ^docs/[^/]*\.svg$
   - id: requirements-txt-fixer
     exclude: >-
-      ^requirements/constraints[.]txt$
+      ^requirements/.*\.txt$
   - id: trailing-whitespace
   - id: file-contents-sorter
     args: ['--ignore-case']

.readthedocs.yml

@@ -15,9 +15,11 @@ build:
   tools:
     python: "3.11"
 
-python:
-  install:
-  - method: pip
-    path: .
-  - requirements: requirements/doc.txt
+  jobs:
+    post_create_environment:
+    - >-
+      pip install
+      . -c requirements/runtime-deps.txt
+      -r requirements/doc.in -c requirements/doc.txt
+
 ...

Makefile

@@ -50,7 +50,7 @@ endif
 	@python -m pip install --upgrade pip
 
 .install-cython: .update-pip $(call to-hash,requirements/cython.txt)
-	@python -m pip install -r requirements/cython.txt -c requirements/constraints.txt
+	@python -m pip install -r requirements/cython.in -c requirements/cython.txt
 	@touch .install-cython
 
 aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py)
@@ -74,7 +74,7 @@ generate-llhttp: .llhttp-gen
 cythonize: .install-cython $(PYXS:.pyx=.c)
 
 .install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS))
-	@python -m pip install -r requirements/dev.txt -c requirements/constraints.txt
+	@python -m pip install -r requirements/dev.in -c requirements/dev.txt
 	@touch .install-deps
 
 .PHONY: lint
@@ -89,7 +89,7 @@ mypy:
 	mypy
 
 .develop: .install-deps generate-llhttp $(call to-hash,$(PYS) $(CYS) $(CS))
-	python -m pip install -e . -c requirements/constraints.txt
+	python -m pip install -e . -c requirements/runtime-deps.txt
 	@touch .develop
 
 .PHONY: test
@@ -177,15 +177,17 @@ doc:
 doc-spelling:
 	@make -C docs spelling SPHINXOPTS="-W --keep-going -n -E"
 
-.PHONY: compile-deps
-compile-deps: .update-pip $(REQS)
-	pip-compile --no-header --allow-unsafe -q --strip-extras \
-		-o requirements/constraints.txt \
-		requirements/constraints.in
-
 .PHONY: install
 install: .update-pip
-	@python -m pip install -r requirements/dev.txt -c requirements/constraints.txt
+	@python -m pip install -r requirements/dev.in -c requirements/dev.txt
 
 .PHONY: install-dev
 install-dev: .develop
+
+.PHONY: sync-direct-runtime-deps
+sync-direct-runtime-deps:
+	@echo Updating 'requirements/runtime-deps.in' from 'setup.cfg'... >&2
+	@echo '# Extracted from `setup.cfg` via `make sync-direct-runtime-deps`' > requirements/runtime-deps.in
+	@echo >> requirements/runtime-deps.in
+	@python -c 'from configparser import ConfigParser; from itertools import chain; from pathlib import Path; cfg = ConfigParser(); cfg.read_string(Path("setup.cfg").read_text()); print("\n".join(line.strip() for line in chain(cfg["options"].get("install_requires").splitlines(), "\n".join(cfg["options.extras_require"].values()).splitlines()) if line.strip()))' \
+		>> requirements/runtime-deps.in

requirements/base.in

@@ -0,0 +1,5 @@
+-r typing-extensions.in
+-r runtime-deps.in
+
+gunicorn
+uvloop; platform_system != "Windows" and implementation_name == "cpython"  # MagicStack/uvloop#14

requirements/base.txt

@@ -1,13 +1,8 @@
--r multidict.txt
--r typing-extensions.txt
-# required c-ares will not build on windows
-aiodns==3.0.0; sys_platform=="linux" or sys_platform=="darwin"
-aiosignal==1.3.1
-async-timeout==4.0.2
-Brotli==1.0.9
-cchardet==2.1.7; python_version < "3.10"  # Unmaintained: aio-libs/aiohttp#6819
-charset-normalizer==3.2.0
-frozenlist==1.4.0
-gunicorn==21.2.0
-uvloop==0.17.0; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.9" # MagicStack/uvloop#14
-yarl==1.9.2
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    pip-compile --allow-unsafe --output-file=requirements/base.txt --strip-extras requirements/multidict.txt
+#
+multidict==6.0.4
+    # via -r requirements/multidict.txt

requirements/broken-projects.in

@@ -0,0 +1,2 @@
+Pillow < 10  # https://github.com/blockdiag/sphinxcontrib-blockdiag/issues/26
+pydantic < 2  # looks like `python-on-whales` hasn't upgraded to support v2

requirements/constraints.in

@@ -1,4 +1,4 @@
--r cython.txt
--r dev.txt
--r doc-spelling.txt
--r lint.txt
+-r cython.in
+-r dev.in
+-r doc-spelling.in
+-r lint.in