From 44b16106c1b3e73858363b2e36a42ae9e8845395 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 25 Apr 2023 18:46:57 -0700 Subject: [PATCH 1/3] Add Support for the Pip 23.1.1 release. --- .github/workflows/ci.yml | 10 +++++----- pex/pip/version.py | 7 +++++++ tox.ini | 3 ++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53fd05231..4773748d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: pip-version: 22_3_1 - os: ubuntu-20.04 python-version: [ 3, 11 ] - pip-version: 23_1 + pip-version: 23_1_1 steps: - name: Calculate Pythons to Expose id: calculate-pythons-to-expose @@ -111,7 +111,7 @@ jobs: - pypy-version: [ 3, 9 ] pip-version: 22_3_1 - pypy-version: [ 3, 9 ] - pip-version: 23_1 + pip-version: 23_1_1 steps: - name: Calculate Pythons to Expose id: calculate-pythons-to-expose @@ -158,7 +158,7 @@ jobs: pip-version: 22_3_1 - os: ubuntu-20.04 python-version: [ 3, 7 ] - pip-version: 23_1 + pip-version: 23_1_1 - os: macos-11 python-version: [ 3, 11 ] pip-version: 20 @@ -170,7 +170,7 @@ jobs: pip-version: 22_3_1 - os: ubuntu-20.04 python-version: [ 3, 11 ] - pip-version: 23_1 + pip-version: 23_1_1 steps: - name: Calculate Pythons to Expose id: calculate-pythons-to-expose @@ -223,7 +223,7 @@ jobs: - pypy-version: [ 3, 9 ] pip-version: 22_3_1 - pypy-version: [ 3, 9 ] - pip-version: 23_1 + pip-version: 23_1_1 steps: - name: Calculate Pythons to Expose id: calculate-pythons-to-expose diff --git a/pex/pip/version.py b/pex/pip/version.py index 925744ac6..39a9ada26 100644 --- a/pex/pip/version.py +++ b/pex/pip/version.py @@ -131,5 +131,12 @@ def values(cls): requires_python=">=3.7", ) + v23_1_1 = PipVersionValue( + version="23.1.1", + setuptools_version="67.7.1", + wheel_version="0.40.0", + requires_python=">=3.7", + ) + VENDORED = v20_3_4_patched LATEST = LatestPipVersion() diff --git a/tox.ini b/tox.ini index bb4dcb8d1..f8bb71f21 100644 --- a/tox.ini +++ b/tox.ini @@ -55,6 +55,7 @@ setenv = pip23_0: _PEX_PIP_VERSION=23.0 pip23_0_1: _PEX_PIP_VERSION=23.0.1 pip23_1: _PEX_PIP_VERSION=23.1 + pip23_1_1: _PEX_PIP_VERSION=23.1.1 # Python 3 (until a fix here in 3.9: https://bugs.python.org/issue13601) switched from stderr # being unbuffered to stderr being buffered by default. This can lead to tests checking stderr # failing to see what they expect if the stderr buffer block has not been flushed. Force stderr @@ -66,7 +67,7 @@ whitelist_externals = bash git -[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,27,35,36,37,38,39,310,311}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-}integration] +[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,27,35,36,37,38,39,310,311}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-}integration] deps = pytest-xdist==1.34.0 {[testenv]deps} From 0292392cb5baf8d73c78a08cd0b14630cc691039 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 25 Apr 2023 21:45:48 -0700 Subject: [PATCH 2/3] Add support for URL quoting. Disambiguate URL quoting from shlex quoting and namespace URL un-quoting as well. --- pex/compatibility.py | 15 +++++++++++---- pex/pip/tool.py | 9 +++++---- pex/requirements.py | 4 ++-- pex/resolve/downloads.py | 9 ++++++--- pex/resolve/locked_resolve.py | 2 +- pex/resolve/resolved_requirement.py | 4 ++-- pex/resolver.py | 4 ++-- 7 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pex/compatibility.py b/pex/compatibility.py index d3d608394..98b68d5cd 100644 --- a/pex/compatibility.py +++ b/pex/compatibility.py @@ -121,7 +121,8 @@ def exec_function(ast, globals_map): if PY3: from urllib import parse as urlparse from urllib.error import HTTPError as HTTPError - from urllib.parse import unquote as unquote + from urllib.parse import quote as _url_quote + from urllib.parse import unquote as _url_unquote from urllib.request import FileHandler as FileHandler from urllib.request import HTTPBasicAuthHandler as HTTPBasicAuthHandler from urllib.request import HTTPDigestAuthHandler as HTTPDigestAuthHandler @@ -131,7 +132,8 @@ def exec_function(ast, globals_map): from urllib.request import Request as Request from urllib.request import build_opener as build_opener else: - from urllib import unquote as unquote + from urllib import quote as _url_quote + from urllib import unquote as _url_unquote import urlparse as urlparse from urllib2 import FileHandler as FileHandler @@ -144,6 +146,9 @@ def exec_function(ast, globals_map): from urllib2 import Request as Request from urllib2 import build_opener as build_opener +url_unquote = _url_unquote +url_quote = _url_quote + if PY3: from queue import Queue as Queue @@ -262,6 +267,8 @@ def append(piece): if PY3: - from shlex import quote as quote + from shlex import quote as _shlex_quote else: - from pipes import quote as quote + from pipes import quote as _shlex_quote + +shlex_quote = _shlex_quote diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 9ecc4322e..f4a51313f 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -14,7 +14,7 @@ from pex import dist_metadata, targets from pex.auth import PasswordEntry from pex.common import safe_mkdir, safe_mkdtemp -from pex.compatibility import get_stderr_bytes_buffer, quote, urlparse +from pex.compatibility import get_stderr_bytes_buffer, shlex_quote, urlparse from pex.interpreter import PythonInterpreter from pex.jobs import Job from pex.network_configuration import NetworkConfiguration @@ -44,7 +44,6 @@ Mapping, Optional, Sequence, - Text, Tuple, ) @@ -353,8 +352,10 @@ def _spawn_pip_isolated( args = self._pip_pex.execute_args(*command) - rendered_env = " ".join("{}={}".format(key, quote(value)) for key, value in env.items()) - rendered_args = " ".join(quote(s) for s in args) + rendered_env = " ".join( + "{}={}".format(key, shlex_quote(value)) for key, value in env.items() + ) + rendered_args = " ".join(shlex_quote(s) for s in args) TRACER.log("Executing: {} {}".format(rendered_env, rendered_args), V=3) return args, subprocess.Popen(args=args, env=env, **popen_kwargs) diff --git a/pex/requirements.py b/pex/requirements.py index f2d23e090..0c4f90ac1 100644 --- a/pex/requirements.py +++ b/pex/requirements.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from pex import attrs, dist_metadata, pex_warnings -from pex.compatibility import unquote, urlparse +from pex.compatibility import url_unquote, urlparse from pex.dist_metadata import ( MetadataError, ProjectNameAndVersion, @@ -485,7 +485,7 @@ def _parse_requirement_line( project_name_and_specifier = _try_parse_project_name_and_specifier_from_path( # There may be whitespace separated markers; so we strip the trailing whitespace # used to support those. - unquote(parsed_url.path).rstrip() + url_unquote(parsed_url.path).rstrip() ) if project_name_and_specifier is not None: project_name = project_name_and_specifier.project_name diff --git a/pex/resolve/downloads.py b/pex/resolve/downloads.py index 9d54408bc..93afb335e 100644 --- a/pex/resolve/downloads.py +++ b/pex/resolve/downloads.py @@ -1,3 +1,6 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + from __future__ import absolute_import import os.path @@ -6,7 +9,7 @@ from pex import hashing from pex.atomic_directory import atomic_directory from pex.common import safe_mkdir, safe_mkdtemp -from pex.compatibility import unquote, urlparse +from pex.compatibility import url_unquote, urlparse from pex.hashing import Sha256 from pex.jobs import Job, Raise, SpawnedJob, execute_parallel from pex.pip.download_observer import DownloadObserver @@ -126,7 +129,7 @@ def _download_and_fingerprint(self, url): download_dir = safe_mkdtemp(prefix="fingerprint_artifact.", dir=downloads) url_info = urlparse.urlparse(url) - src_file = urlparse.unquote(url_info.path) + src_file = url_unquote(url_info.path) temp_dest = os.path.join(download_dir, os.path.basename(src_file)) if url_info.scheme == "file": @@ -173,7 +176,7 @@ def download( url_info = urlparse.urlparse(artifact.url) if url_info.scheme == "file": - src_file = unquote(url_info.path) + src_file = url_unquote(url_info.path) try: shutil.copy(src_file, dest_file) except (IOError, OSError) as e: diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index 04920d8d9..bf6966cc1 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -8,7 +8,7 @@ from collections import OrderedDict, defaultdict, deque from pex.common import pluralize -from pex.compatibility import unquote, urlparse +from pex.compatibility import url_unquote, urlparse from pex.dist_metadata import DistMetadata, Requirement from pex.enum import Enum from pex.orderedset import OrderedSet diff --git a/pex/resolve/resolved_requirement.py b/pex/resolve/resolved_requirement.py index cca7626f4..db6775190 100644 --- a/pex/resolve/resolved_requirement.py +++ b/pex/resolve/resolved_requirement.py @@ -6,7 +6,7 @@ import hashlib from pex import hashing -from pex.compatibility import unquote, urlparse +from pex.compatibility import url_unquote, urlparse from pex.dist_metadata import ProjectNameAndVersion, Requirement from pex.hashing import HashlibHasher from pex.pep_440 import Version @@ -88,7 +88,7 @@ def parse(cls, url): raw_url=url, normalized_url=normalized_url, scheme=parse_scheme(url_info.scheme) if url_info.scheme else None, - path=unquote(url_info.path), + path=url_unquote(url_info.path), ) raw_url = attr.ib(eq=False) # type: str diff --git a/pex/resolver.py b/pex/resolver.py index e7f7e5d53..457c2c478 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -17,7 +17,7 @@ from pex.atomic_directory import AtomicDirectory, atomic_directory from pex.auth import PasswordEntry from pex.common import safe_mkdir, safe_mkdtemp -from pex.compatibility import unquote, urlparse +from pex.compatibility import url_unquote, urlparse from pex.dist_metadata import DistMetadata, Distribution, ProjectNameAndVersion, Requirement from pex.fingerprinted_distribution import FingerprintedDistribution from pex.jobs import Raise, SpawnedJob, execute_parallel @@ -711,7 +711,7 @@ def _resolve_direct_file_deps( urlinfo = urlparse.urlparse(requirement.url) if urlinfo.scheme != "file": continue - dist_path = unquote(urlinfo.path).rstrip() + dist_path = url_unquote(urlinfo.path).rstrip() if not os.path.exists(dist_path): raise Unsatisfiable( "The {wheel} wheel has a dependency on {url} which does not exist on this " From 48fc54325b3b8085b58fcd7cbfc450c000c9ed38 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 25 Apr 2023 21:47:09 -0700 Subject: [PATCH 3/3] Fix CI bitrot due to Pants 2.13.0.dev3 deletion. The 2.12.0.dev3 release was deleted from PyPI to make room for the latest batch of Pants releases; so switch to the Pants S3 bucket find links repo. --- tests/integration/test_issue_1730.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_issue_1730.py b/tests/integration/test_issue_1730.py index 6e468ff1d..937c71960 100644 --- a/tests/integration/test_issue_1730.py +++ b/tests/integration/test_issue_1730.py @@ -7,6 +7,7 @@ import pytest +from pex.compatibility import url_quote from pex.testing import IS_PYPY, PY_VER, run_pex_command from pex.typing import TYPE_CHECKING @@ -28,6 +29,16 @@ def test_check_install_issue_1730( with open(constraints, "w") as fp: print("packaging==21.3", file=fp) + # The PyPI-hosted version of Pants 2.13.0.dev3 was deleted to make space available; so we use + # the Pants S3 bucket instead. + pants_hosted_version = "2.12.0.dev3+git552439fc" + find_links = ( + "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/" + "552439fc4500284f97d09461d8f9a89df1ac1676/" + "{pants_hosted_version}/" + "index.html" + ).format(pants_hosted_version=url_quote(pants_hosted_version)) + pex_args = [ "--pex-root", pex_root, @@ -35,6 +46,8 @@ def test_check_install_issue_1730( pex_root, "--constraints", constraints, + "-f", + find_links, "pantsbuild.pants.testutil==2.12.0.dev3", "--", "-c", @@ -45,9 +58,9 @@ def test_check_install_issue_1730( old_result.assert_failure() assert ( "Failed to resolve compatible distributions:\n" - "1: pantsbuild.pants.testutil==2.12.0.dev3 requires pantsbuild.pants==2.12.0.dev3 but " - "pantsbuild.pants 2.12.0.dev3 was resolved" in old_result.error - ), old_result.error + "1: pantsbuild.pants.testutil=={version} requires pantsbuild.pants=={version} but " + "pantsbuild.pants {version} was resolved" + ).format(version=pants_hosted_version) in old_result.error, old_result.error new_result = run_pex_command(args=pex_args, quiet=True) new_result.assert_success()