Merge branch 'main' into standard-dist-name-check

.coveragerc

@@ -1,5 +1,6 @@
 [run]
 branch = True
+dynamic_context = test_function
 
 source =
     warehouse
@@ -24,3 +25,6 @@ omit =
 exclude_lines =
     pragma: no cover
     class \w+\(Interface\):
+
+# Don't show us anything that's already 100% covered.
+skip_covered = True

.github/dependabot.yml

@@ -10,3 +10,6 @@ updates:
   - dependency-type: direct
   - dependency-type: indirect
   rebase-strategy: "disabled"
+  ignore:
+  # Always ignore elasticsearch, future versions are always incompatible with our provider
+  - dependency-name: "elasticsearch"

.github/workflows/ci.yml

@@ -1,5 +1,12 @@
 name: CI
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
 jobs:
   test:
     strategy:
@@ -14,19 +21,19 @@ jobs:
             needs-python: ${{ true }}
             needs-node: ${{ false }}
           - name: Documentation
-            command: make docs BINDIR="$(dirname $(which python))"
+            command: bin/docs
             needs-python: ${{ true }}
             needs-node: ${{ false }}
           - name: Dependencies
-            command: make github-actions-deps BINDIR="$(dirname $(which python))"
+            command: bin/github-actions-deps
             needs-python: ${{ true }}
             needs-node: ${{ false }}
           - name: Licenses
-            command: make licenses BINDIR="$(dirname $(which python))"
+            command: bin/licenses
             needs-python: ${{ true }}
             needs-node: ${{ false }}
           - name: Translations
-            command: make translations BINDIR="$(dirname $(which python))"
+            command: bin/translations
             needs-python: ${{ true }}
             needs-node: ${{ false }}
           - name: Static Tests
@@ -55,10 +62,14 @@ jobs:
         run: |
           sudo apt -y update
           sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config
+      - name: Read .python-version
+        if: ${{ matrix.needs-python }}
+        run: |
+           echo "WAREHOUSE_PYTHON_VERSION=$(<.python-version)" >> $GITHUB_ENV
       - uses: actions/setup-python@v2
         if: ${{ matrix.needs-python }}
         with:
-          python-version: 3.8.2
+          python-version: ${{ env.WAREHOUSE_PYTHON_VERSION }}
       - name: Cache Python dependencies
         if: ${{ matrix.needs-python }}
         uses: actions/cache@v2

.python-version

@@ -1 +1 @@
-3.8.2
+3.9.10

.readthedocs.yml

@@ -9,3 +9,9 @@ version: 2
 python:
   install:
     - requirements: requirements/docs.txt
+
+# Use the same version of Python we use here
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.8"

Dockerfile

@@ -44,7 +44,7 @@ RUN gulp dist
 
 # Now we're going to build our actual application, but not the actual production
 # image that it gets deployed into.
-FROM python:3.8.2-slim-buster as build
+FROM python:3.9.10-slim-buster as build
 
 # Define whether we're building a production or a development image. This will
 # generally be used to control whether or not we install our development and
@@ -103,7 +103,7 @@ RUN set -x \
             install --no-deps --no-binary hiredis \
                     -r /tmp/requirements/deploy.txt \
                     -r /tmp/requirements/main.txt \
-                    $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
+                    $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt -r /tmp/requirements/docs.txt'; fi) \
     && find /opt/warehouse -name '*.pyc' -delete
 
 
@@ -112,7 +112,7 @@ RUN set -x \
 
 # Now we're going to build our actual application image, which will eventually
 # pull in the static files that were built above.
-FROM python:3.8.2-slim-buster
+FROM python:3.9.10-slim-buster
 
 # Setup some basic environment variables that are ~never going to change.
 ENV PYTHONUNBUFFERED 1
@@ -138,7 +138,7 @@ RUN set -x \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
         libpq5 libxml2 libxslt1.1 libcurl4  \
-        $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client'; fi) \
+        $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev'; fi) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

Makefile

@@ -1,39 +1,12 @@
-BINDIR = $(PWD)/.state/env/bin
-GITHUB_ACTIONS := $(shell echo "$${GITHUB_ACTIONS:-false}")
-GITHUB_BASE_REF := $(shell echo "$${GITHUB_BASE_REF:-false}")
 DB := example
 IPYTHON := no
-LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
 # needed in development environment
 ifeq ($(WAREHOUSE_IPYTHON_SHELL), 1)
     IPYTHON = yes
 endif
 
-define DEPCHECKER
-import sys
-from pip_api import parse_requirements
-
-left, right = sys.argv[1:3]
-left_reqs = parse_requirements(left).keys()
-right_reqs = parse_requirements(right).keys()
-
-extra_in_left = left_reqs - right_reqs
-extra_in_right = right_reqs - left_reqs
-
-if extra_in_left:
-	for dep in sorted(extra_in_left):
-		print("- {}".format(dep))
-
-if extra_in_right:
-	for dep in sorted(extra_in_right):
-		print("+ {}".format(dep))
-
-if extra_in_left or extra_in_right:
-	sys.exit(1)
-endef
-
 default:
 	@echo "Call a specific subcommand:"
 	@echo
@@ -44,29 +17,25 @@ default:
 	@echo
 	@exit 1
 
-.state/env/pyvenv.cfg: requirements/dev.txt requirements/docs.txt requirements/lint.txt requirements/ipython.txt
-	# Create our Python 3.8 virtual environment
-	rm -rf .state/env
-	python3.8 -m venv .state/env
+.state/docker-build-web: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt requirements/lint.txt requirements/docs.txt requirements/dev.txt requirements/tests.txt
+	# Build our web container for this project.
+	docker-compose build --build-arg IPYTHON=$(IPYTHON) --force-rm web
 
-	# install/upgrade general requirements
-	.state/env/bin/python -m pip install --upgrade pip setuptools wheel
+	# Mark the state so we don't rebuild this needlessly.
+	mkdir -p .state
+	touch .state/docker-build-web
 
-	# install various types of requirements
-	.state/env/bin/python -m pip install -r requirements/dev.txt
-	.state/env/bin/python -m pip install -r requirements/docs.txt
-	.state/env/bin/python -m pip install -r requirements/lint.txt
+.state/docker-build-static: Dockerfile package.json package-lock.json .babelrc
+	# Build our static container for this project.
+	docker-compose build --force-rm static
 
-	# install ipython if enabled
-ifeq ($(IPYTHON),"yes")
-	.state/env/bin/python -m pip install -r requirements/ipython.txt
-endif
+	# Mark the state so we don't rebuild this needlessly.
+	mkdir -p .state
+	touch .state/docker-build-static
 
-.state/docker-build: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt
-	# Build our docker containers for this project.
-	docker-compose build --build-arg IPYTHON=$(IPYTHON) --force-rm web
+.state/docker-build: .state/docker-build-web .state/docker-build-static
+	# Build the worker container for this project
 	docker-compose build --force-rm worker
-	docker-compose build --force-rm static
 
 	# Mark the state so we don't rebuild this needlessly.
 	mkdir -p .state
@@ -80,78 +49,52 @@ build:
 serve: .state/docker-build
 	docker-compose up --remove-orphans
 
-debug: .state/docker-build
+debug: .state/docker-build-web
 	docker-compose run --rm --service-ports web
 
-tests: .state/docker-build
-	docker-compose run --rm web env -i ENCODING="C.UTF-8" \
-								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
-								  bin/tests --postgresql-host db $(T) $(TESTARGS)
-
-static_tests: .state/docker-build
-	docker-compose run --rm static env -i ENCODING="C.UTF-8" \
-								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
-								  bin/static_tests $(T) $(TESTARGS)
-
-static_pipeline: .state/docker-build
-	docker-compose run --rm static env -i ENCODING="C.UTF-8" \
-								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
-								  bin/static_pipeline $(T) $(TESTARGS)
-
-reformat: .state/docker-build
-	docker-compose run --rm web env -i ENCODING="C.UTF-8" \
-								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
-								  bin/reformat
-
-lint: .state/docker-build
-	docker-compose run --rm web env -i ENCODING="C.UTF-8" \
-								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
-								  bin/lint && bin/static_lint
-
-docs: .state/env/pyvenv.cfg
-	$(MAKE) -C docs/ doctest SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build"
-	$(MAKE) -C docs/ html SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build"
-
-licenses:
-	bin/licenses
-
-export DEPCHECKER
-deps: .state/env/pyvenv.cfg
-	$(eval TMPDIR := $(shell mktemp -d))
-	$(BINDIR)/pip-compile --upgrade --allow-unsafe -o $(TMPDIR)/deploy.txt requirements/deploy.in > /dev/null
-	$(BINDIR)/pip-compile --upgrade --allow-unsafe -o $(TMPDIR)/main.txt requirements/main.in > /dev/null
-	$(BINDIR)/pip-compile --upgrade --allow-unsafe -o $(TMPDIR)/lint.txt requirements/lint.in > /dev/null
-	echo "$$DEPCHECKER" | $(BINDIR)/python - $(TMPDIR)/deploy.txt requirements/deploy.txt
-	echo "$$DEPCHECKER" | $(BINDIR)/python - $(TMPDIR)/main.txt requirements/main.txt
-	echo "$$DEPCHECKER" | $(BINDIR)/python - $(TMPDIR)/lint.txt requirements/lint.txt
-	rm -r $(TMPDIR)
-	$(BINDIR)/pip check
-
-requirements/%.txt: requirements/%.in .state/env/pyvenv.cfg
-	$(BINDIR)/pip-compile --allow-unsafe --generate-hashes --output-file=$@ $<
-
-github-actions-deps:
-ifneq ($(GITHUB_BASE_REF), false)
-	git fetch origin $(GITHUB_BASE_REF):refs/remotes/origin/$(GITHUB_BASE_REF)
-	# Check that the following diff will exit with 0 or 1
-	git diff --name-only FETCH_HEAD || test $? -le 1 || exit 1
-	# Make the dependencies if any changed files are requirements files, otherwise exit
-	git diff --name-only FETCH_HEAD | grep '^requirements/' || exit 0 && $(MAKE) deps
-endif
+tests: .state/docker-build-web
+	docker-compose run --rm web bin/tests --postgresql-host db $(T) $(TESTARGS)
+
+static_tests: .state/docker-build-static
+	docker-compose run --rm static bin/static_tests $(T) $(TESTARGS)
+
+static_pipeline: .state/docker-build-static
+	docker-compose run --rm static bin/static_pipeline $(T) $(TESTARGS)
+
+reformat: .state/docker-build-web
+	docker-compose run --rm web bin/reformat
+
+lint: .state/docker-build-web
+	docker-compose run --rm web bin/lint && bin/static_lint
+
+docs: .state/docker-build-web
+	docker-compose run --rm web bin/docs
 
-initdb:
+licenses: .state/docker-build-web
+	docker-compose run --rm web bin/licenses
+
+deps: .state/docker-build-web
+	docker-compose run --rm web bin/deps
+
+translations: .state/docker-build-web
+	docker-compose run --rm web bin/translations
+
+requirements/%.txt: requirements/%.in
+	docker-compose run --rm web bin/pip-compile --allow-unsafe --generate-hashes --output-file=$@ $<
+
+initdb: .state/docker-build-web
 	docker-compose run --rm web psql -h db -d postgres -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse';"
 	docker-compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS warehouse"
 	docker-compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE warehouse ENCODING 'UTF8'"
-	xz -d -f -k dev/$(DB).sql.xz --stdout | docker-compose run --rm web psql -h db -d warehouse -U postgres -v ON_ERROR_STOP=1 -1 -f -
+	docker-compose run --rm web bash -c "xz -d -f -k dev/$(DB).sql.xz --stdout | psql -h db -d warehouse -U postgres -v ON_ERROR_STOP=1 -1 -f -"
 	docker-compose run --rm web python -m warehouse db upgrade head
+	docker-compose run --rm web python -m warehouse sponsors populate-db
 	$(MAKE) reindex
-	docker-compose run web python -m warehouse sponsors populate-db
 
-reindex:
+reindex: .state/docker-build-web
 	docker-compose run --rm web python -m warehouse search reindex
 
-shell:
+shell: .state/docker-build-web
 	docker-compose run --rm web python -m warehouse shell
 
 clean:
@@ -164,40 +107,4 @@ purge: stop clean
 stop:
 	docker-compose down -v
 
-compile-pot: .state/env/pyvenv.cfg
-	PYTHONPATH=$(PWD) $(BINDIR)/pybabel extract \
-		-F babel.cfg \
-		--omit-header \
-		--output="warehouse/locale/messages.pot" \
-		warehouse
-
-init-po: .state/env/pyvenv.cfg
-	$(BINDIR)/pybabel init \
-		--input-file="warehouse/locale/messages.pot" \
-		--output-dir="warehouse/locale/" \
-		--locale="$(L)"
-
-update-po: .state/env/pyvenv.cfg
-	$(BINDIR)/pybabel update \
-		--input-file="warehouse/locale/messages.pot" \
-		--output-file="warehouse/locale/$(L)/LC_MESSAGES/messages.po" \
-		--locale="$(L)"
-
-compile-po: .state/env/pyvenv.cfg
-	$(BINDIR)/pybabel compile \
-		--input-file="warehouse/locale/$(L)/LC_MESSAGES/messages.po" \
-		--directory="warehouse/locale/" \
-		--locale="$(L)"
-
-build-mos: compile-pot
-	for LOCALE in $(LOCALES) ; do \
-		L=$$LOCALE $(MAKE) compile-po ; \
-		done
-
-translations: compile-pot
-ifneq ($(GITHUB_ACTIONS), false)
-	git diff --quiet ./warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1)
-else
-endif
-
 .PHONY: default build serve initdb shell tests docs deps clean purge debug stop compile-pot

bin/depchecker.py

@@ -0,0 +1,33 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.import sys
+
+import sys
+
+from pip_api import parse_requirements
+
+left, right = sys.argv[1:3]
+left_reqs = parse_requirements(left).keys()
+right_reqs = parse_requirements(right).keys()
+
+extra_in_left = left_reqs - right_reqs
+extra_in_right = right_reqs - left_reqs
+
+if extra_in_left:
+    for dep in sorted(extra_in_left):
+        print("- {}".format(dep))
+
+if extra_in_right:
+    for dep in sorted(extra_in_right):
+        print("+ {}".format(dep))
+
+if extra_in_left or extra_in_right:
+    sys.exit(1)