Merge branch 'pypa:main' into joe/warm_cache_in_threadpool

.github/workflows/ci.yml

@@ -167,24 +167,13 @@ jobs:
  with:
  python-version: ${{ matrix.python }}
 
- # We use a RAMDisk on Windows, since filesystem IO is a big slowdown
- # for our tests.
- - name: Create a RAMDisk
- run: ./tools/ci/New-RAMDisk.ps1 -Drive R -Size 1GB
-
- - name: Setup RAMDisk permissions
- run: |
- mkdir R:\Temp
- $acl = Get-Acl "R:\Temp"
- $rule = New-Object System.Security.AccessControl.FileSystemAccessRule(
- "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow"
- )
- $acl.AddAccessRule($rule)
- Set-Acl "R:\Temp" $acl
-
+ # We use C:\Temp (which is already available on the worker)
+ # as a temporary directory for all of the tests because the
+ # default value (under the user dir) is more deeply nested
+ # and causes tests to fail with "path too long" errors.
  - run: pip install nox
  env:
- TEMP: "R:\\Temp"
+ TEMP: "C:\\Temp"
 
  # Main check
  - name: Run unit tests
@@ -194,7 +183,7 @@ jobs:
  -m unit
  --verbose --numprocesses auto --showlocals
  env:
- TEMP: "R:\\Temp"
+ TEMP: "C:\\Temp"
 
  - name: Run integration tests (group 1)
  if: matrix.group == 1
@@ -203,7 +192,7 @@ jobs:
  -m integration -k "not test_install"
  --verbose --numprocesses auto --showlocals
  env:
- TEMP: "R:\\Temp"
+ TEMP: "C:\\Temp"
 
  - name: Run integration tests (group 2)
  if: matrix.group == 2
@@ -212,7 +201,7 @@ jobs:
  -m integration -k "test_install"
  --verbose --numprocesses auto --showlocals
  env:
- TEMP: "R:\\Temp"
+ TEMP: "C:\\Temp"
 
  tests-zipapp:
  name: tests / zipapp

.pre-commit-config.yaml

@@ -21,22 +21,11 @@ repos:
  hooks:
  - id: black
 
-- repo: https://github.com/PyCQA/flake8
- rev: 6.0.0
+- repo: https://github.com/astral-sh/ruff-pre-commit
+ # Ruff version.
+ rev: v0.0.270
  hooks:
- - id: flake8
- additional_dependencies: [
- 'flake8-bugbear',
- 'flake8-logging-format',
- 'flake8-implicit-str-concat',
- ]
- exclude: tests/data
-
-- repo: https://github.com/PyCQA/isort
- rev: 5.12.0
- hooks:
- - id: isort
- files: \.py$
+ - id: ruff
 
 - repo: https://github.com/pre-commit/mirrors-mypy
  rev: v0.961

NEWS.rst

@@ -53,7 +53,7 @@ Deprecations and Removals
  ``--config-settings``. (`#11859 <https://github.com/pypa/pip/issues/11859>`_)
 - Using ``--config-settings`` with projects that don't have a ``pyproject.toml`` now prints
  a deprecation warning. In the future the presence of config settings will automatically
- enable the default build backend for legacy projects and pass the setttings to it. (`#11915 <https://github.com/pypa/pip/issues/11915>`_)
+ enable the default build backend for legacy projects and pass the settings to it. (`#11915 <https://github.com/pypa/pip/issues/11915>`_)
 - Remove ``setup.py install`` fallback when building a wheel failed for projects without
  ``pyproject.toml``. (`#8368 <https://github.com/pypa/pip/issues/8368>`_)
 - When the ``wheel`` package is not installed, pip now uses the default build backend

news/11847.bugfix.rst

@@ -0,0 +1 @@
+Prevent downloading files twice when PEP 658 metadata is present

news/11920.bugfix.rst

@@ -0,0 +1 @@
+Add permission check before configuration

news/11996.process.rst

@@ -0,0 +1 @@
+Deprecate support for eggs for Python 3.11 or later, when the new ``importlib.metadata`` backend is used to load distribution metadata. This only affects the egg *distribution format* (with the ``.egg`` extension); distributions using the ``.egg-info`` *metadata format* (but are not actually eggs) are not affected. For more information about eggs, see `relevant section in the setuptools documentation <https://setuptools.pypa.io/en/stable/deprecated/python_eggs.html>`__.

news/12042.bugfix.rst

@@ -0,0 +1 @@
+Correctly parse ``dist-info-metadata`` values from JSON-format index data.

news/12063.removal.rst

@@ -0,0 +1,2 @@
+Deprecate legacy version and version specifiers that don't conform to `PEP 440
+<https://peps.python.org/pep-0440/>`_

news/12067.bugfix.rst

@@ -0,0 +1 @@
+Fail with an error if the ``--python`` option is specified after the subcommand name.

news/12079.bugfix.rst

@@ -0,0 +1 @@
+Fix slowness when using ``importlib.metadata`` (the default way for pip to read metadata in Python 3.11+) and there is a large overlap between already installed and to-be-installed packages.

news/12119.bugfix.rst

@@ -0,0 +1,3 @@
+Pass the ``-r`` flag to mercurial to be explicit that a revision is passed and protect
+against ``hg`` options injection as part of VCS URLs. Users that do not have control on
+VCS URLs passed to pip are advised to upgrade.

news/4256.removal.rst

@@ -0,0 +1,4 @@
+``freeze`` no longer excludes the ``setuptools``, ``distribute``, and ``wheel``
+from the output when running on Python 3.12 or later, where they are not
+included in a virtual environment by default. Use ``--exclude`` if you wish to
+exclude any of these packages.

news/certifi.vendor.rst

@@ -0,0 +1 @@
+Upgrade certifi to 2023.5.7

news/platformdirs.vendor.rst

@@ -0,0 +1 @@
+Upgrade platformdirs to 3.8.1

news/pygments.vendor.rst

@@ -0,0 +1 @@
+Upgrade pygments to 2.15.1

news/pyparsing.vendor.rst

@@ -0,0 +1 @@
+Upgrade pyparsing to 3.1.0

news/rich.vendor.rst

@@ -0,0 +1 @@
+Upgrade rich to 13.4.2

news/setuptools.vendor.rst

@@ -0,0 +1 @@
+Upgrade setuptools to 68.0.0

news/typing_extensions.vendor.rst

@@ -0,0 +1 @@
+Upgrade typing_extensions to 4.7.1

news/urllib3.vendor.rst

@@ -0,0 +1 @@
+Upgrade urllib3 to 1.26.16

noxfile.py

@@ -219,7 +219,7 @@ def pinned_requirements(path: Path) -> Iterator[Tuple[str, str]]:
  new_version = old_version
  for inner_name, inner_version in pinned_requirements(vendor_txt):
  if inner_name == name:
- # this is a dedicated assignment, to make flake8 happy
+ # this is a dedicated assignment, to make lint happy
  new_version = inner_version
  break
  else:

pyproject.toml

@@ -71,3 +71,38 @@ setuptools = "pkg_resources"
 CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt"
 distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt"
 webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE"
+
+[tool.ruff]
+extend-exclude = [
+ "./build",
+ ".scratch",
+ "_vendor",
+ "data",
+]
+ignore = [
+ "B019",
+ "B020",
+ "B904", # Ruff enables opinionated warnings by default
+ "B905", # Ruff enables opinionated warnings by default
+ "G202",
+]
+line-length = 88
+select = [
+ "B",
+ "E",
+ "F",
+ "W",
+ "G",
+ "ISC",
+ "I",
+]
+
+[tool.ruff.per-file-ignores]
+"noxfile.py" = ["G"]
+"tests/*" = ["B011"]
+
+[tool.ruff.isort]
+# We need to explicitly make pip "first party" as it's imported by code in
+# the docs and tests directories.
+known-first-party = ["pip"]
+known-third-party = ["pip._vendor"]

src/pip/_internal/cli/base_command.py

@@ -131,6 +131,17 @@ def _main(self, args: List[str]) -> int:
  ", ".join(sorted(always_enabled_features)),
  )
 
+ # Make sure that the --python argument isn't specified after the
+ # subcommand. We can tell, because if --python was specified,
+ # we should only reach this point if we're running in the created
+ # subprocess, which has the _PIP_RUNNING_IN_SUBPROCESS environment
+ # variable set.
+ if options.python and "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ:
+ logger.critical(
+ "The --python option must be placed before the pip subcommand name"
+ )
+ sys.exit(ERROR)
+
  # TODO: Try to get these passing down from the command?
  # without resorting to os.environ to hold these.
  # This also affects isolated builds and it should.

src/pip/_internal/commands/check.py

@@ -7,6 +7,7 @@
 from pip._internal.operations.check import (
  check_package_set,
  create_package_set_from_installed,
+ warn_legacy_versions_and_specifiers,
 )
 from pip._internal.utils.misc import write_output
 
@@ -21,6 +22,7 @@ class CheckCommand(Command):
 
  def run(self, options: Values, args: List[str]) -> int:
  package_set, parsing_probs = create_package_set_from_installed()
+ warn_legacy_versions_and_specifiers(package_set)
  missing, conflicting = check_package_set(package_set)
 
  for project_name in missing:

src/pip/_internal/commands/download.py

@@ -130,6 +130,7 @@ def run(self, options: Values, args: List[str]) -> int:
  self.trace_basic_info(finder)
 
  requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+ requirement_set.warn_legacy_versions_and_specifiers()
 
  downloaded: List[str] = []
  for req in requirement_set.requirements.values():

src/pip/_internal/commands/freeze.py

@@ -1,14 +1,25 @@
 import sys
 from optparse import Values
-from typing import List
+from typing import AbstractSet, List
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import SUCCESS
 from pip._internal.operations.freeze import freeze
 from pip._internal.utils.compat import stdlib_pkgs
 
-DEV_PKGS = {"pip", "setuptools", "distribute", "wheel"}
+
+def _should_suppress_build_backends() -> bool:
+ return sys.version_info < (3, 12)
+
+
+def _dev_pkgs() -> AbstractSet[str]:
+ pkgs = {"pip"}
+
+ if _should_suppress_build_backends():
+ pkgs |= {"setuptools", "distribute", "wheel"}
+
+ return pkgs
 
 
 class FreezeCommand(Command):
@@ -61,7 +72,7 @@ def add_options(self) -> None:
  action="store_true",
  help=(
  "Do not skip these packages in the output:"
- " {}".format(", ".join(DEV_PKGS))
+ " {}".format(", ".join(_dev_pkgs()))
  ),
  )
  self.cmd_opts.add_option(
@@ -77,7 +88,7 @@ def add_options(self) -> None:
  def run(self, options: Values, args: List[str]) -> int:
  skip = set(stdlib_pkgs)
  if not options.freeze_all:
- skip.update(DEV_PKGS)
+ skip.update(_dev_pkgs())
 
  if options.excludes:
  skip.update(options.excludes)

src/pip/_internal/commands/install.py

@@ -387,6 +387,9 @@ def run(self, options: Values, args: List[str]) -> int:
  json.dump(report.to_dict(), f, indent=2, ensure_ascii=False)
 
  if options.dry_run:
+ # In non dry-run mode, the legacy versions and specifiers check
+ # will be done as part of conflict detection.
+ requirement_set.warn_legacy_versions_and_specifiers()
  would_install_items = sorted(
  (r.metadata["name"], r.metadata["version"])
  for r in requirement_set.requirements_to_install

src/pip/_internal/commands/wheel.py

@@ -145,6 +145,7 @@ def run(self, options: Values, args: List[str]) -> int:
  self.trace_basic_info(finder)
 
  requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+ requirement_set.warn_legacy_versions_and_specifiers()
 
  reqs_to_build: List[InstallRequirement] = []
  for req in requirement_set.requirements.values():

src/pip/_internal/configuration.py

@@ -210,8 +210,15 @@ def save(self) -> None:
  # Ensure directory exists.
  ensure_dir(os.path.dirname(fname))
 
- with open(fname, "w") as f:
- parser.write(f)
+ # Ensure directory's permission(need to be writeable)
+ try:
+ with open(fname, "w") as f:
+ parser.write(f)
+ except OSError as error:
+ raise ConfigurationError(
+ f"An error occurred while writing to the configuration file "
+ f"{fname}: {error}"
+ )
 
  #
  # Private routines

src/pip/_internal/metadata/importlib/_envs.py

@@ -151,7 +151,7 @@ def _emit_egg_deprecation(location: Optional[str]) -> None:
  deprecated(
  reason=f"Loading egg at {location} is deprecated.",
  replacement="to use pip for package installation.",
- gone_in=None,
+ gone_in="23.3",
  )
 
 
@@ -174,7 +174,7 @@ def _iter_distributions(self) -> Iterator[BaseDistribution]:
  for location in self._paths:
  yield from finder.find(location)
  for dist in finder.find_eggs(location):
- # _emit_egg_deprecation(dist.location) # TODO: Enable this.
+ _emit_egg_deprecation(dist.location)
  yield dist
  # This must go last because that's how pkg_resources tie-breaks.
  yield from finder.find_linked(location)