From 8295c9941b6516ccaaba9c91af025cb2e373dba0 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 24 Apr 2023 12:29:05 -0600 Subject: [PATCH 01/70] Deprecate .egg in the imporlib-metadata backend This provides us a path to remove all pkg_resources usages on Python 3.11 or later, and thus avoid the problem that pkg_resources uses Python API deprecated in 3.12. --- news/11996.process.rst | 1 + src/pip/_internal/metadata/importlib/_envs.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 news/11996.process.rst diff --git a/news/11996.process.rst b/news/11996.process.rst new file mode 100644 index 00000000000..d585bd39183 --- /dev/null +++ b/news/11996.process.rst @@ -0,0 +1 @@ +Deprecate support for eggs for Python 3.11 or later, when the new ``importlib.metadata`` backend is used to load distribution metadata. This only affects the egg *distribution format* (with the ``.egg`` extension); distributions using the ``.egg-info`` *metadata format* (but are not actually eggs) are not affected. For more information about eggs, see `relevant section in the setuptools documentation `__. diff --git a/src/pip/_internal/metadata/importlib/_envs.py b/src/pip/_internal/metadata/importlib/_envs.py index cbec59e2c6d..3850ddaf412 100644 --- a/src/pip/_internal/metadata/importlib/_envs.py +++ b/src/pip/_internal/metadata/importlib/_envs.py @@ -151,7 +151,7 @@ def _emit_egg_deprecation(location: Optional[str]) -> None: deprecated( reason=f"Loading egg at {location} is deprecated.", replacement="to use pip for package installation.", - gone_in=None, + gone_in="23.3", ) @@ -174,7 +174,7 @@ def _iter_distributions(self) -> Iterator[BaseDistribution]: for location in self._paths: yield from finder.find(location) for dist in finder.find_eggs(location): - # _emit_egg_deprecation(dist.location) # TODO: Enable this. + _emit_egg_deprecation(dist.location) yield dist # This must go last because that's how pkg_resources tie-breaks. yield from finder.find_linked(location) From 8d381eeec23cd1a0901266b189740d1a20f08534 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 2 Jun 2023 11:03:49 +0100 Subject: [PATCH 02/70] Warn if the --python option is not specified before the subcommand name --- news/12067.bugfix.rst | 1 + src/pip/_internal/cli/base_command.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 news/12067.bugfix.rst diff --git a/news/12067.bugfix.rst b/news/12067.bugfix.rst new file mode 100644 index 00000000000..84f2d235e79 --- /dev/null +++ b/news/12067.bugfix.rst @@ -0,0 +1 @@ +Warn if the ``--python`` option is specified after the subcommand name. diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 637fba18cfc..87e6cf6deaf 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -131,6 +131,18 @@ def _main(self, args: List[str]) -> int: ", ".join(sorted(always_enabled_features)), ) + + # Make sure that the --python argument isn't specified after the + # subcommand. We can tell, because if --python was specified, + # we should only reach this point if we're running in the created + # subprocess, which has the _PIP_RUNNING_IN_SUBPROCESS environment + # variable set. + if options.python and "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ: + logger.warning( + "The --python option is ignored if placed after " + "the pip subcommand name" + ) + # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. # This also affects isolated builds and it should. From 6aa4d48b23e11f6a837b61f4c3fda1a247758357 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 11:53:43 +0100 Subject: [PATCH 03/70] Fix the test failure --- tests/functional/test_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 63712827479..8559d93684b 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1157,7 +1157,7 @@ def test_install_nonlocal_compatible_wheel( "--find-links", data.find_links, "--only-binary=:all:", - "--python", + "--python-version", "3", "--platform", "fakeplat", @@ -1177,7 +1177,7 @@ def test_install_nonlocal_compatible_wheel( "--find-links", data.find_links, "--only-binary=:all:", - "--python", + "--python-version", "3", "--platform", "fakeplat", From 073666e2994896a66eff3a474dac3cf03b2dfdd9 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 11:57:04 +0100 Subject: [PATCH 04/70] Make this an error rather than a warning --- news/12067.bugfix.rst | 2 +- src/pip/_internal/cli/base_command.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/news/12067.bugfix.rst b/news/12067.bugfix.rst index 84f2d235e79..87d76bc2b06 100644 --- a/news/12067.bugfix.rst +++ b/news/12067.bugfix.rst @@ -1 +1 @@ -Warn if the ``--python`` option is specified after the subcommand name. +Fail with an error if the ``--python`` option is specified after the subcommand name. diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 87e6cf6deaf..5130a45053f 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -138,10 +138,10 @@ def _main(self, args: List[str]) -> int: # subprocess, which has the _PIP_RUNNING_IN_SUBPROCESS environment # variable set. if options.python and "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ: - logger.warning( - "The --python option is ignored if placed after " - "the pip subcommand name" + logger.critical( + "The --python option must be placed before the pip subcommand name" ) + sys.exit(ERROR) # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. From 16f145d30657093dab2fe66cd3695d248c46db45 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 12:03:22 +0100 Subject: [PATCH 05/70] Add a test for the error --- tests/functional/test_python_option.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/functional/test_python_option.py b/tests/functional/test_python_option.py index 8bf16d7a56b..ca124933e12 100644 --- a/tests/functional/test_python_option.py +++ b/tests/functional/test_python_option.py @@ -39,3 +39,14 @@ def test_python_interpreter( script.pip("--python", env_path, "uninstall", "simplewheel", "--yes") result = script.pip("--python", env_path, "list", "--format=json") assert json.loads(result.stdout) == before + +def test_error_python_option_wrong_location( + script: PipTestEnvironment, + tmpdir: Path, + shared_data: TestData, +) -> None: + env_path = os.fspath(tmpdir / "venv") + env = EnvBuilder(with_pip=False) + env.create(env_path) + + script.pip("list", "--python", env_path, "--format=json", expect_error=True) From de8f0b5ed17fc59fab1f20e9cca92c24ca89136d Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 12:11:41 +0100 Subject: [PATCH 06/70] Lint --- src/pip/_internal/cli/base_command.py | 1 - tests/functional/test_python_option.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 5130a45053f..6a3b8e6c213 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -131,7 +131,6 @@ def _main(self, args: List[str]) -> int: ", ".join(sorted(always_enabled_features)), ) - # Make sure that the --python argument isn't specified after the # subcommand. We can tell, because if --python was specified, # we should only reach this point if we're running in the created diff --git a/tests/functional/test_python_option.py b/tests/functional/test_python_option.py index ca124933e12..ecfd819eb7c 100644 --- a/tests/functional/test_python_option.py +++ b/tests/functional/test_python_option.py @@ -40,6 +40,7 @@ def test_python_interpreter( result = script.pip("--python", env_path, "list", "--format=json") assert json.loads(result.stdout) == before + def test_error_python_option_wrong_location( script: PipTestEnvironment, tmpdir: Path, From 80cb6f443fca2c51b19ec4c3853b6261d2f4b6e6 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:26:05 +0100 Subject: [PATCH 07/70] Switch to ruff for linting --- .pre-commit-config.yaml | 13 ++++--------- noxfile.py | 2 +- pyproject.toml | 30 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fc455b9d64..dd2fc623522 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,16 +21,11 @@ repos: hooks: - id: black -- repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.270 hooks: - - id: flake8 - additional_dependencies: [ - 'flake8-bugbear', - 'flake8-logging-format', - 'flake8-implicit-str-concat', - ] - exclude: tests/data + - id: ruff - repo: https://github.com/PyCQA/isort rev: 5.12.0 diff --git a/noxfile.py b/noxfile.py index 565a5039955..ee03447d359 100644 --- a/noxfile.py +++ b/noxfile.py @@ -219,7 +219,7 @@ def pinned_requirements(path: Path) -> Iterator[Tuple[str, str]]: new_version = old_version for inner_name, inner_version in pinned_requirements(vendor_txt): if inner_name == name: - # this is a dedicated assignment, to make flake8 happy + # this is a dedicated assignment, to make lint happy new_version = inner_version break else: diff --git a/pyproject.toml b/pyproject.toml index 139c37e18d7..18c990ef42d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,3 +71,33 @@ setuptools = "pkg_resources" CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt" distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt" webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE" + +[tool.ruff] +exclude = [ + "./build", + ".nox", + ".tox", + ".scratch", + "_vendor", + "data", +] +ignore = [ + "B019", + "B020", + "B904", # Ruff enables opinionated warnings by default + "B905", # Ruff enables opinionated warnings by default + "G202", +] +line-length = 88 +select = [ + "B", + "E", + "F", + "W", + "G", + "ISC", +] + +[tool.ruff.per-file-ignores] +"noxfile.py" = ["G"] +"tests/*" = ["B011"] From 9824a426d466680d132aa027add88ec0dda9116f Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:32:44 +0100 Subject: [PATCH 08/70] Fix new lint errors --- src/pip/_internal/network/auth.py | 4 +++- tests/lib/__init__.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index c0efa765c85..94a82fa6618 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -514,7 +514,9 @@ def handle_401(self, resp: Response, **kwargs: Any) -> Response: # Consume content and release the original connection to allow our new # request to reuse the same one. - resp.content + # The result of the assignment isn't used, it's just needed to consume + # the content. + _ = resp.content resp.raw.release_conn() # Add our new username and password to the request diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 7410072f50e..2c6dbafb901 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -684,7 +684,9 @@ def run( # Pass expect_stderr=True to allow any stderr. We do this because # we do our checking of stderr further on in check_stderr(). kw["expect_stderr"] = True - result = super().run(cwd=cwd, *args, **kw) + # Ignore linter check + # B026 Star-arg unpacking after a keyword argument is strongly discouraged + result = super().run(cwd=cwd, *args, **kw) # noqa if expect_error and not allow_error: if result.returncode == 0: From 0af7d6de7adfdc2a0c3b5854cac8bccca3f7e887 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:36:34 +0100 Subject: [PATCH 09/70] Tidy up file exclusions --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 18c990ef42d..5f5dfa6414e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,10 +73,8 @@ distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt" webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE" [tool.ruff] -exclude = [ +extend-exclude = [ "./build", - ".nox", - ".tox", ".scratch", "_vendor", "data", From 7c3418b2d03aeae665a0a0f01b584ffba4bc57ea Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:50:10 +0100 Subject: [PATCH 10/70] Add explicit ID to noqa comment Co-authored-by: q0w <43147888+q0w@users.noreply.github.com> --- tests/lib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 2c6dbafb901..7c06feaf38c 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -686,7 +686,7 @@ def run( kw["expect_stderr"] = True # Ignore linter check # B026 Star-arg unpacking after a keyword argument is strongly discouraged - result = super().run(cwd=cwd, *args, **kw) # noqa + result = super().run(cwd=cwd, *args, **kw) # noqa: B026 if expect_error and not allow_error: if result.returncode == 0: From bdef9159bd3734381fe90a907750c8e168170c38 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 15:06:11 +0100 Subject: [PATCH 11/70] Use ruff for import sorting as well --- .pre-commit-config.yaml | 6 ------ pyproject.toml | 7 +++++++ tests/functional/test_install_compat.py | 8 ++++++-- tests/functional/test_install_upgrade.py | 9 +++++++-- tests/functional/test_install_user.py | 2 +- tests/functional/test_install_vcs_git.py | 2 +- tests/functional/test_wheel.py | 7 +++++-- 7 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd2fc623522..b0aef0d60b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,12 +27,6 @@ repos: hooks: - id: ruff -- repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - files: \.py$ - - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.961 hooks: diff --git a/pyproject.toml b/pyproject.toml index 5f5dfa6414e..b7c0d154598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,8 +94,15 @@ select = [ "W", "G", "ISC", + "I", ] [tool.ruff.per-file-ignores] "noxfile.py" = ["G"] "tests/*" = ["B011"] + +[tool.ruff.isort] +# We need to explicitly make pip "first party" as it's imported by code in +# the docs and tests directories. +known-first-party = ["pip"] +known-third-party = ["pip._vendor"] diff --git a/tests/functional/test_install_compat.py b/tests/functional/test_install_compat.py index ae27ebd536e..8374d487b1f 100644 --- a/tests/functional/test_install_compat.py +++ b/tests/functional/test_install_compat.py @@ -7,8 +7,12 @@ import pytest -from tests.lib import pyversion # noqa: F401 -from tests.lib import PipTestEnvironment, TestData, assert_all_changes +from tests.lib import ( + PipTestEnvironment, + TestData, + assert_all_changes, + pyversion, # noqa: F401 +) @pytest.mark.network diff --git a/tests/functional/test_install_upgrade.py b/tests/functional/test_install_upgrade.py index fc61d70bc5e..09c01d7eb18 100644 --- a/tests/functional/test_install_upgrade.py +++ b/tests/functional/test_install_upgrade.py @@ -6,8 +6,13 @@ import pytest -from tests.lib import pyversion # noqa: F401 -from tests.lib import PipTestEnvironment, ResolverVariant, TestData, assert_all_changes +from tests.lib import ( + PipTestEnvironment, + ResolverVariant, + TestData, + assert_all_changes, + pyversion, # noqa: F401 +) from tests.lib.local_repos import local_checkout from tests.lib.wheel import make_wheel diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 9bdadb94203..3cae4a467e9 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -8,12 +8,12 @@ import pytest -from tests.lib import pyversion # noqa: F401 from tests.lib import ( PipTestEnvironment, TestData, create_basic_wheel_for_package, need_svn, + pyversion, # noqa: F401 ) from tests.lib.local_repos import local_checkout from tests.lib.venv import VirtualEnvironment diff --git a/tests/functional/test_install_vcs_git.py b/tests/functional/test_install_vcs_git.py index 971526c5181..2abc7aa0fd2 100644 --- a/tests/functional/test_install_vcs_git.py +++ b/tests/functional/test_install_vcs_git.py @@ -3,11 +3,11 @@ import pytest -from tests.lib import pyversion # noqa: F401 from tests.lib import ( PipTestEnvironment, _change_test_package_version, _create_test_package, + pyversion, # noqa: F401 ) from tests.lib.git_submodule_helpers import ( _change_test_package_submodule, diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index 1e3e90e410f..cfaef541dcf 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -7,8 +7,11 @@ import pytest from pip._internal.cli.status_codes import ERROR -from tests.lib import pyversion # noqa: F401 -from tests.lib import PipTestEnvironment, TestData +from tests.lib import ( + PipTestEnvironment, + TestData, + pyversion, # noqa: F401 +) def add_files_to_dist_directory(folder: Path) -> None: From 16df8cdcbf895f707fe6f83bdceed8a09315018b Mon Sep 17 00:00:00 2001 From: Robert Pollak Date: Wed, 7 Jun 2023 16:04:20 +0200 Subject: [PATCH 12/70] NEWS.rst typo 'setttings' should be 'settings'. --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index b0ae642634d..f24aaaa4094 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -53,7 +53,7 @@ Deprecations and Removals ``--config-settings``. (`#11859 `_) - Using ``--config-settings`` with projects that don't have a ``pyproject.toml`` now prints a deprecation warning. In the future the presence of config settings will automatically - enable the default build backend for legacy projects and pass the setttings to it. (`#11915 `_) + enable the default build backend for legacy projects and pass the settings to it. (`#11915 `_) - Remove ``setup.py install`` fallback when building a wheel failed for projects without ``pyproject.toml``. (`#8368 `_) - When the ``wheel`` package is not installed, pip now uses the default build backend From 6c3db098ff0b6e537157eff53b1aba79fa7fa4b0 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 20:58:40 +0100 Subject: [PATCH 13/70] Fix parsing of JSON index dist-info-metadata values --- src/pip/_internal/models/link.py | 97 +++++++++++++++++++++----------- tests/unit/test_collector.py | 37 +++++++----- 2 files changed, 89 insertions(+), 45 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index e741c3283cd..ee3045166bb 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -69,18 +69,6 @@ class LinkHash: def __post_init__(self) -> None: assert self.name in _SUPPORTED_HASHES - @classmethod - def parse_pep658_hash(cls, dist_info_metadata: str) -> Optional["LinkHash"]: - """Parse a PEP 658 data-dist-info-metadata hash.""" - if dist_info_metadata == "true": - return None - name, sep, value = dist_info_metadata.partition("=") - if not sep: - return None - if name not in _SUPPORTED_HASHES: - return None - return cls(name=name, value=value) - @classmethod @functools.lru_cache(maxsize=None) def find_hash_url_fragment(cls, url: str) -> Optional["LinkHash"]: @@ -107,6 +95,20 @@ def is_hash_allowed(self, hashes: Optional[Hashes]) -> bool: return hashes.is_hash_allowed(self.name, hex_digest=self.value) +@dataclass(frozen=True) +class MetadataFile: + """Information about a core metadata file associated with a distribution.""" + + hashes: Optional[dict[str, str]] + + # TODO: Do we care about stripping out unsupported hash methods? + def __init__(self, hashes: Optional[dict[str, str]]): + if hashes: + hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} + # We need to use this as this is a frozen dataclass + object.__setattr__(self, "hashes", hashes) + + def _clean_url_path_part(part: str) -> str: """ Clean a "part" of a URL path (i.e. after splitting on "@" characters). @@ -179,7 +181,7 @@ class Link(KeyBasedCompareMixin): "comes_from", "requires_python", "yanked_reason", - "dist_info_metadata", + "metadata_file_data", "cache_link_parsing", "egg_fragment", ] @@ -190,7 +192,7 @@ def __init__( comes_from: Optional[Union[str, "IndexContent"]] = None, requires_python: Optional[str] = None, yanked_reason: Optional[str] = None, - dist_info_metadata: Optional[str] = None, + metadata_file_data: Optional[MetadataFile] = None, cache_link_parsing: bool = True, hashes: Optional[Mapping[str, str]] = None, ) -> None: @@ -208,11 +210,10 @@ def __init__( a simple repository HTML link. If the file has been yanked but no reason was provided, this should be the empty string. See PEP 592 for more information and the specification. - :param dist_info_metadata: the metadata attached to the file, or None if no such - metadata is provided. This is the value of the "data-dist-info-metadata" - attribute, if present, in a simple repository HTML link. This may be parsed - into its own `Link` by `self.metadata_link()`. See PEP 658 for more - information and the specification. + :param metadata_file_data: the metadata attached to the file, or None if + no such metadata is provided. This argument, if not None, indicates + that a separate metadata file exists, and also optionally supplies + hashes for that file. :param cache_link_parsing: A flag that is used elsewhere to determine whether resources retrieved from this link should be cached. PyPI URLs should generally have this set to False, for example. @@ -220,6 +221,10 @@ def __init__( determine the validity of a download. """ + # The comes_from, requires_python, and metadata_file_data arguments are + # only used by classmethods of this class, and are not used in client + # code directly. + # url can be a UNC windows share if url.startswith("\\\\"): url = path_to_url(url) @@ -239,7 +244,7 @@ def __init__( self.comes_from = comes_from self.requires_python = requires_python if requires_python else None self.yanked_reason = yanked_reason - self.dist_info_metadata = dist_info_metadata + self.metadata_file_data = metadata_file_data super().__init__(key=url, defining_class=Link) @@ -262,9 +267,20 @@ def from_json( url = _ensure_quoted_url(urllib.parse.urljoin(page_url, file_url)) pyrequire = file_data.get("requires-python") yanked_reason = file_data.get("yanked") - dist_info_metadata = file_data.get("dist-info-metadata") hashes = file_data.get("hashes", {}) + # The dist-info-metadata value may be a boolean, or a dict of hashes. + metadata_info = file_data.get("dist-info-metadata", False) + if isinstance(metadata_info, dict): + # The file exists, and hashes have been supplied + metadata_file_data = MetadataFile(metadata_info) + elif metadata_info: + # The file exists, but there are no hashes + metadata_file_data = MetadataFile(None) + else: + # The file does not exist + metadata_file_data = None + # The Link.yanked_reason expects an empty string instead of a boolean. if yanked_reason and not isinstance(yanked_reason, str): yanked_reason = "" @@ -278,7 +294,7 @@ def from_json( requires_python=pyrequire, yanked_reason=yanked_reason, hashes=hashes, - dist_info_metadata=dist_info_metadata, + metadata_file_data=metadata_file_data, ) @classmethod @@ -298,14 +314,35 @@ def from_element( url = _ensure_quoted_url(urllib.parse.urljoin(base_url, href)) pyrequire = anchor_attribs.get("data-requires-python") yanked_reason = anchor_attribs.get("data-yanked") - dist_info_metadata = anchor_attribs.get("data-dist-info-metadata") + + # The dist-info-metadata value may be the string "true", or a string of + # the form "hashname=hashval" + metadata_info = anchor_attribs.get("data-dist-info-metadata") + if metadata_info == "true": + # The file exists, but there are no hashes + metadata_file_data = MetadataFile(None) + elif metadata_info is None: + # The file does not exist + metadata_file_data = None + else: + # The file exists, and hashes have been supplied + hashname, sep, hashval = metadata_info.partition("=") + if sep == "=": + metadata_file_data = MetadataFile({hashname: hashval}) + else: + # Error - data is wrong. Treat as no hashes supplied. + logger.debug( + "Index returned invalid data-dist-info-metadata value: %s", + metadata_info, + ) + metadata_file_data = MetadataFile(None) return cls( url, comes_from=page_url, requires_python=pyrequire, yanked_reason=yanked_reason, - dist_info_metadata=dist_info_metadata, + metadata_file_data=metadata_file_data, ) def __str__(self) -> str: @@ -407,17 +444,13 @@ def subdirectory_fragment(self) -> Optional[str]: return match.group(1) def metadata_link(self) -> Optional["Link"]: - """Implementation of PEP 658 parsing.""" - # Note that Link.from_element() parsing the "data-dist-info-metadata" attribute - # from an HTML anchor tag is typically how the Link.dist_info_metadata attribute - # gets set. - if self.dist_info_metadata is None: + """Return a link to the associated core metadata file (if any).""" + if self.metadata_file_data is None: return None metadata_url = f"{self.url_without_fragment}.metadata" - metadata_link_hash = LinkHash.parse_pep658_hash(self.dist_info_metadata) - if metadata_link_hash is None: + if self.metadata_file_data.hashes is None: return Link(metadata_url) - return Link(metadata_url, hashes=metadata_link_hash.as_dict()) + return Link(metadata_url, hashes=self.metadata_file_data.hashes) def as_hashes(self) -> Hashes: return Hashes({k: [v] for k, v in self._hashes.items()}) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index e855d78e126..b3c9fcf1f5b 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -30,6 +30,7 @@ from pip._internal.models.link import ( Link, LinkHash, + MetadataFile, _clean_url_path, _ensure_quoted_url, ) @@ -527,7 +528,7 @@ def test_parse_links_json() -> None: requires_python=">=3.7", yanked_reason=None, hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, - dist_info_metadata="sha512=aabdd41", + metadata_file_data=MetadataFile({"sha512": "aabdd41"}), ), ] @@ -603,12 +604,12 @@ def test_parse_links__yanked_reason(anchor_html: str, expected: Optional[str]) - ), ], ) -def test_parse_links__dist_info_metadata( +def test_parse_links__metadata_file_data( anchor_html: str, expected: Optional[str], hashes: Dict[str, str], ) -> None: - link = _test_parse_links_data_attribute(anchor_html, "dist_info_metadata", expected) + link = _test_parse_links_data_attribute(anchor_html, "metadata_file_data", expected) assert link._hashes == hashes @@ -1080,17 +1081,27 @@ def test_link_hash_parsing(url: str, result: Optional[LinkHash]) -> None: @pytest.mark.parametrize( - "dist_info_metadata, result", + "metadata_attrib, expected", [ - ("sha256=aa113592bbe", LinkHash("sha256", "aa113592bbe")), - ("sha256=", LinkHash("sha256", "")), - ("sha500=aa113592bbe", None), - ("true", None), - ("", None), - ("aa113592bbe", None), + ("sha256=aa113592bbe", MetadataFile({"sha256": "aa113592bbe"})), + ("sha256=", MetadataFile({"sha256": ""})), + ("sha500=aa113592bbe", MetadataFile({})), + ("true", MetadataFile(None)), + (None, None), + # TODO: Are these correct? + ("", MetadataFile(None)), + ("aa113592bbe", MetadataFile(None)), ], ) -def test_pep658_hash_parsing( - dist_info_metadata: str, result: Optional[LinkHash] +def test_metadata_file_info_parsing_html( + metadata_attrib: str, expected: Optional[MetadataFile] ) -> None: - assert LinkHash.parse_pep658_hash(dist_info_metadata) == result + attribs: Dict[str, Optional[str]] = { + "href": "something", + "data-dist-info-metadata": metadata_attrib, + } + page_url = "dummy_for_comes_from" + base_url = "https://index.url/simple" + link = Link.from_element(attribs, page_url, base_url) + assert link is not None and link.metadata_file_data == expected + # TODO: Do we need to do something for the JSON data? From cc554edab8897749c87495333018754080d06781 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:01:10 +0100 Subject: [PATCH 14/70] Add a news file --- news/12042.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12042.bugfix.rst diff --git a/news/12042.bugfix.rst b/news/12042.bugfix.rst new file mode 100644 index 00000000000..34d97743540 --- /dev/null +++ b/news/12042.bugfix.rst @@ -0,0 +1 @@ +Correctly parse ``dist-info-metadata`` values from JSON-format index data. From 8f89997d0dad1644b258297e2e3b9cc70d44e51d Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:11:34 +0100 Subject: [PATCH 15/70] Fix types to be 3.7-compatible --- src/pip/_internal/models/link.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index ee3045166bb..9630448bcfb 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -99,10 +99,10 @@ def is_hash_allowed(self, hashes: Optional[Hashes]) -> bool: class MetadataFile: """Information about a core metadata file associated with a distribution.""" - hashes: Optional[dict[str, str]] + hashes: Optional[Dict[str, str]] # TODO: Do we care about stripping out unsupported hash methods? - def __init__(self, hashes: Optional[dict[str, str]]): + def __init__(self, hashes: Optional[Dict[str, str]]): if hashes: hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} # We need to use this as this is a frozen dataclass From cfb4923d5d016dc58dc4e4b896992c476f0ddce8 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:21:32 +0100 Subject: [PATCH 16/70] Fix bad test data in test_parse_links_json --- tests/unit/test_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index b3c9fcf1f5b..838dd2efb88 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -492,7 +492,7 @@ def test_parse_links_json() -> None: "url": "/files/holygrail-1.0-py3-none-any.whl", "hashes": {"sha256": "sha256 hash", "blake2b": "blake2b hash"}, "requires-python": ">=3.7", - "dist-info-metadata": "sha512=aabdd41", + "dist-info-metadata": {"sha512": "aabdd41"}, }, ], } From 93b274eee79b9c114728f0864a29751ee7698fca Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:44:48 +0100 Subject: [PATCH 17/70] Missed a change to one of the tests --- tests/unit/test_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 838dd2efb88..513e4b1347b 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -587,19 +587,19 @@ def test_parse_links__yanked_reason(anchor_html: str, expected: Optional[str]) - # Test with value "true". ( '', - "true", + MetadataFile(None), {}, ), # Test with a provided hash value. ( '', # noqa: E501 - "sha256=aa113592bbe", + MetadataFile({"sha256": "aa113592bbe"}), {}, ), # Test with a provided hash value for both the requirement as well as metadata. ( '', # noqa: E501 - "sha256=aa113592bbe", + MetadataFile({"sha256": "aa113592bbe"}), {"sha512": "abc132409cb"}, ), ], From 232cc9dd5284fbc7554bcd291bf14e31413da78a Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 8 Jun 2023 09:52:51 +0100 Subject: [PATCH 18/70] Parse hash data before passing to MetadataFile --- src/pip/_internal/models/link.py | 24 ++++++++++++++++-------- tests/unit/test_collector.py | 5 ++--- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 9630448bcfb..7b45f3f3ed4 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -101,12 +101,20 @@ class MetadataFile: hashes: Optional[Dict[str, str]] - # TODO: Do we care about stripping out unsupported hash methods? - def __init__(self, hashes: Optional[Dict[str, str]]): - if hashes: - hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} - # We need to use this as this is a frozen dataclass - object.__setattr__(self, "hashes", hashes) + def __post_init__(self) -> None: + if self.hashes is not None: + assert all(name in _SUPPORTED_HASHES for name in self.hashes) + + +def supported_hashes(hashes: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]: + # Remove any unsupported hash types from the mapping. If this leaves no + # supported hashes, return None + if hashes is None: + return None + hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} + if len(hashes) > 0: + return hashes + return None def _clean_url_path_part(part: str) -> str: @@ -273,7 +281,7 @@ def from_json( metadata_info = file_data.get("dist-info-metadata", False) if isinstance(metadata_info, dict): # The file exists, and hashes have been supplied - metadata_file_data = MetadataFile(metadata_info) + metadata_file_data = MetadataFile(supported_hashes(metadata_info)) elif metadata_info: # The file exists, but there are no hashes metadata_file_data = MetadataFile(None) @@ -328,7 +336,7 @@ def from_element( # The file exists, and hashes have been supplied hashname, sep, hashval = metadata_info.partition("=") if sep == "=": - metadata_file_data = MetadataFile({hashname: hashval}) + metadata_file_data = MetadataFile(supported_hashes({hashname: hashval})) else: # Error - data is wrong. Treat as no hashes supplied. logger.debug( diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 513e4b1347b..d1e68fab76f 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -1085,10 +1085,10 @@ def test_link_hash_parsing(url: str, result: Optional[LinkHash]) -> None: [ ("sha256=aa113592bbe", MetadataFile({"sha256": "aa113592bbe"})), ("sha256=", MetadataFile({"sha256": ""})), - ("sha500=aa113592bbe", MetadataFile({})), + ("sha500=aa113592bbe", MetadataFile(None)), ("true", MetadataFile(None)), (None, None), - # TODO: Are these correct? + # Attribute is present but invalid ("", MetadataFile(None)), ("aa113592bbe", MetadataFile(None)), ], @@ -1104,4 +1104,3 @@ def test_metadata_file_info_parsing_html( base_url = "https://index.url/simple" link = Link.from_element(attribs, page_url, base_url) assert link is not None and link.metadata_file_data == expected - # TODO: Do we need to do something for the JSON data? From 5168881b438b2851ae4c9459a8c06beee2058639 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 8 Jun 2023 10:10:15 +0100 Subject: [PATCH 19/70] Implement PEP 714 - rename dist-info-metadata --- src/pip/_internal/models/link.py | 19 +++++++++--- tests/unit/test_collector.py | 53 +++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 7b45f3f3ed4..3cfc3e8c4fe 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -277,8 +277,13 @@ def from_json( yanked_reason = file_data.get("yanked") hashes = file_data.get("hashes", {}) - # The dist-info-metadata value may be a boolean, or a dict of hashes. - metadata_info = file_data.get("dist-info-metadata", False) + # PEP 714: Indexes must use the name core-metadata, but + # clients should support the old name as a fallback for compatibility. + metadata_info = file_data.get("core-metadata") + if metadata_info is None: + metadata_info = file_data.get("dist-info-metadata") + + # The metadata info value may be a boolean, or a dict of hashes. if isinstance(metadata_info, dict): # The file exists, and hashes have been supplied metadata_file_data = MetadataFile(supported_hashes(metadata_info)) @@ -286,7 +291,7 @@ def from_json( # The file exists, but there are no hashes metadata_file_data = MetadataFile(None) else: - # The file does not exist + # False or not present: the file does not exist metadata_file_data = None # The Link.yanked_reason expects an empty string instead of a boolean. @@ -323,9 +328,13 @@ def from_element( pyrequire = anchor_attribs.get("data-requires-python") yanked_reason = anchor_attribs.get("data-yanked") - # The dist-info-metadata value may be the string "true", or a string of + # PEP 714: Indexes must use the name data-core-metadata, but + # clients should support the old name as a fallback for compatibility. + metadata_info = anchor_attribs.get("data-core-metadata") + if metadata_info is None: + metadata_info = anchor_attribs.get("data-dist-info-metadata") + # The metadata info value may be the string "true", or a string of # the form "hashname=hashval" - metadata_info = anchor_attribs.get("data-dist-info-metadata") if metadata_info == "true": # The file exists, but there are no hashes metadata_file_data = MetadataFile(None) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index d1e68fab76f..5410a4afc03 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -486,7 +486,15 @@ def test_parse_links_json() -> None: "requires-python": ">=3.7", "dist-info-metadata": False, }, - # Same as above, but parsing dist-info-metadata. + # Same as above, but parsing core-metadata. + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + "requires-python": ">=3.7", + "core-metadata": {"sha512": "aabdd41"}, + }, + # Ensure fallback to dist-info-metadata works { "filename": "holygrail-1.0-py3-none-any.whl", "url": "/files/holygrail-1.0-py3-none-any.whl", @@ -494,6 +502,15 @@ def test_parse_links_json() -> None: "requires-python": ">=3.7", "dist-info-metadata": {"sha512": "aabdd41"}, }, + # Ensure that core-metadata gets priority. + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + "requires-python": ">=3.7", + "core-metadata": {"sha512": "aabdd41"}, + "dist-info-metadata": {"sha512": "this_is_wrong"}, + }, ], } ).encode("utf8") @@ -530,6 +547,22 @@ def test_parse_links_json() -> None: hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, metadata_file_data=MetadataFile({"sha512": "aabdd41"}), ), + Link( + "https://example.com/files/holygrail-1.0-py3-none-any.whl", + comes_from=page.url, + requires_python=">=3.7", + yanked_reason=None, + hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + metadata_file_data=MetadataFile({"sha512": "aabdd41"}), + ), + Link( + "https://example.com/files/holygrail-1.0-py3-none-any.whl", + comes_from=page.url, + requires_python=">=3.7", + yanked_reason=None, + hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + metadata_file_data=MetadataFile({"sha512": "aabdd41"}), + ), ] # Ensure the metadata info can be parsed into the correct link. @@ -586,22 +619,34 @@ def test_parse_links__yanked_reason(anchor_html: str, expected: Optional[str]) - ), # Test with value "true". ( - '', + '', MetadataFile(None), {}, ), # Test with a provided hash value. ( - '', # noqa: E501 + '', # noqa: E501 MetadataFile({"sha256": "aa113592bbe"}), {}, ), # Test with a provided hash value for both the requirement as well as metadata. ( - '', # noqa: E501 + '', # noqa: E501 MetadataFile({"sha256": "aa113592bbe"}), {"sha512": "abc132409cb"}, ), + # Ensure the fallback to the old name works. + ( + '', # noqa: E501 + MetadataFile({"sha256": "aa113592bbe"}), + {}, + ), + # Ensure that the data-core-metadata name gets priority. + ( + '', # noqa: E501 + MetadataFile({"sha256": "aa113592bbe"}), + {}, + ), ], ) def test_parse_links__metadata_file_data( From 3eb3ddd873dfde2c146fcc5c82b2fa0ba363ac69 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Thu, 8 Jun 2023 17:57:17 +0200 Subject: [PATCH 20/70] Fix slowness on Python 3.11 when updating an existing large environment. --- news/12079.bugfix.rst | 1 + src/pip/_internal/resolution/resolvelib/candidates.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 news/12079.bugfix.rst diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst new file mode 100644 index 00000000000..79496798adc --- /dev/null +++ b/news/12079.bugfix.rst @@ -0,0 +1 @@ +Fix slowness on Python 3.11 when updating an existing large environment. diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 31020e27ad1..de04e1d73f2 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -341,6 +341,7 @@ def __init__( self.dist = dist self._ireq = _make_install_req_from_dist(dist, template) self._factory = factory + self._version = None # This is just logging some messages, so we can do it eagerly. # The returned dist would be exactly the same as self.dist because we @@ -376,7 +377,9 @@ def name(self) -> str: @property def version(self) -> CandidateVersion: - return self.dist.version + if self._version is None: + self._version = self.dist.version + return self._version @property def is_editable(self) -> bool: From 1269d0d240f4d22ed1134bb854bf2177a82a8d66 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Mon, 12 Jun 2023 16:31:28 +0200 Subject: [PATCH 21/70] Update news snippet. --- news/12079.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst index 79496798adc..d65c8ee7698 100644 --- a/news/12079.bugfix.rst +++ b/news/12079.bugfix.rst @@ -1 +1 @@ -Fix slowness on Python 3.11 when updating an existing large environment. +Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. From 1a80e41504008c8f3b63b5fb59c6b7476fcae3b9 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Mon, 12 Jun 2023 21:43:34 +0200 Subject: [PATCH 22/70] Mention in changelog that 12079 is an issue mostly in Python 3.11. --- news/12079.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst index d65c8ee7698..b5b7e553bf3 100644 --- a/news/12079.bugfix.rst +++ b/news/12079.bugfix.rst @@ -1 +1 @@ -Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. +Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. This is the default in Python 3.11, though it can be overridden with the ``_PIP_USE_IMPORTLIB_METADATA`` environment variable. From 6aef9326d93dee61cb69cfa8770ca445891fa990 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Mon, 12 Jun 2023 21:49:12 +0200 Subject: [PATCH 23/70] Update news/12079.bugfix.rst Co-authored-by: Tzu-ping Chung --- news/12079.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst index b5b7e553bf3..5ee05026808 100644 --- a/news/12079.bugfix.rst +++ b/news/12079.bugfix.rst @@ -1 +1 @@ -Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. This is the default in Python 3.11, though it can be overridden with the ``_PIP_USE_IMPORTLIB_METADATA`` environment variable. +Fix slowness when using ``importlib.metadata`` (the default way for pip to read metadata in Python 3.11+) and there is a large overlap between already installed and to-be-installed packages. From 67deaf7576a9aa14f37bd6ebef5bdce038069b60 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:36:15 +0800 Subject: [PATCH 24/70] Add permission check before configuration --- src/pip/_internal/configuration.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 8fd46c9b8e0..35790369a25 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -210,8 +210,16 @@ def save(self) -> None: # Ensure directory exists. ensure_dir(os.path.dirname(fname)) - with open(fname, "w") as f: - parser.write(f) + # Ensure directory's permission(need to be writeable) + if os.access(fname, os.W_OK): + with open(fname, "w") as f: + parser.write(f) + else: + raise ConfigurationError( + "Configuation file not writeable".format( + ": ".join(fname) + ) + ) # # Private routines From 19b41050efdcb2b8e74c16fded3064abb96f95fa Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:00:03 +0800 Subject: [PATCH 25/70] Add permission check before configuration --- news/11920.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11920.bugfix.rst diff --git a/news/11920.bugfix.rst b/news/11920.bugfix.rst new file mode 100644 index 00000000000..f91667c5251 --- /dev/null +++ b/news/11920.bugfix.rst @@ -0,0 +1 @@ +Add permission check before configuration \ No newline at end of file From dafca3f7e3985ce2f2351d168efbd91fa5b2f4b2 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:11:20 +0800 Subject: [PATCH 26/70] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 35790369a25..f8de9239704 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,10 +216,8 @@ def save(self) -> None: parser.write(f) else: raise ConfigurationError( - "Configuation file not writeable".format( - ": ".join(fname) + "Configuation file not writeable {}".format(': '.join(fname)) ) - ) # # Private routines From f77661e478d3169e43fd2ba540a0c9778e924a45 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:35:46 +0800 Subject: [PATCH 27/70] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index f8de9239704..ebcfe7df0f8 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,10 +211,10 @@ def save(self) -> None: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - if os.access(fname, os.W_OK): + try: with open(fname, "w") as f: parser.write(f) - else: + except: raise ConfigurationError( "Configuation file not writeable {}".format(': '.join(fname)) ) From f74650725b7fa57a5b17f2b839624fa36622ba7e Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:36:15 +0800 Subject: [PATCH 28/70] Add permission check before configuration --- src/pip/_internal/configuration.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 8fd46c9b8e0..35790369a25 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -210,8 +210,16 @@ def save(self) -> None: # Ensure directory exists. ensure_dir(os.path.dirname(fname)) - with open(fname, "w") as f: - parser.write(f) + # Ensure directory's permission(need to be writeable) + if os.access(fname, os.W_OK): + with open(fname, "w") as f: + parser.write(f) + else: + raise ConfigurationError( + "Configuation file not writeable".format( + ": ".join(fname) + ) + ) # # Private routines From 920bcd0c631be649317b5a1e00019ad029353d5c Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:00:03 +0800 Subject: [PATCH 29/70] Add permission check before configuration --- news/11920.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11920.bugfix.rst diff --git a/news/11920.bugfix.rst b/news/11920.bugfix.rst new file mode 100644 index 00000000000..f91667c5251 --- /dev/null +++ b/news/11920.bugfix.rst @@ -0,0 +1 @@ +Add permission check before configuration \ No newline at end of file From 2dbda58efc6bfd0b9115626d294ab95b11712e9a Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:11:20 +0800 Subject: [PATCH 30/70] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 35790369a25..f8de9239704 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,10 +216,8 @@ def save(self) -> None: parser.write(f) else: raise ConfigurationError( - "Configuation file not writeable".format( - ": ".join(fname) + "Configuation file not writeable {}".format(': '.join(fname)) ) - ) # # Private routines From 17147b8fd36a851da7897da26eef8f68aec364a0 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:35:46 +0800 Subject: [PATCH 31/70] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index f8de9239704..ebcfe7df0f8 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,10 +211,10 @@ def save(self) -> None: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - if os.access(fname, os.W_OK): + try: with open(fname, "w") as f: parser.write(f) - else: + except: raise ConfigurationError( "Configuation file not writeable {}".format(': '.join(fname)) ) From 5986dd27c5797245c70daf922f89bddc16ee656e Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Fri, 16 Jun 2023 21:35:49 +0800 Subject: [PATCH 32/70] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index ebcfe7df0f8..f8de9239704 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,10 +211,10 @@ def save(self) -> None: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - try: + if os.access(fname, os.W_OK): with open(fname, "w") as f: parser.write(f) - except: + else: raise ConfigurationError( "Configuation file not writeable {}".format(': '.join(fname)) ) From 05e936aecb6427403c6a99dcbcefa63514b98425 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Sat, 17 Jun 2023 12:16:25 +0800 Subject: [PATCH 33/70] Add permission check before configuration7 --- src/pip/_internal/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index f8de9239704..ba34886b6b1 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -345,7 +345,7 @@ def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]: # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) + ) or not os.access(config_file[kinds.SITE], os.W_OK) if should_load_user_config: # The legacy config file is overridden by the new config file yield kinds.USER, config_files[kinds.USER] From 8747268d44250b164957f3a7671bd30ef9f250c7 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Sat, 17 Jun 2023 12:30:36 +0800 Subject: [PATCH 34/70] Add permission check before configuration8 --- src/pip/_internal/configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index ba34886b6b1..0a7d183ae8b 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -345,7 +345,8 @@ def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]: # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) or not os.access(config_file[kinds.SITE], os.W_OK) + ) or not os.access(config_files[kinds.SITE], os.W_OK) + if should_load_user_config: # The legacy config file is overridden by the new config file yield kinds.USER, config_files[kinds.USER] From 81c8a3ffbca1aca2c7bfae1433d0dd52050fc9b3 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Sat, 17 Jun 2023 12:42:47 +0800 Subject: [PATCH 35/70] Add permission check before configuration9 --- src/pip/_internal/configuration.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 0a7d183ae8b..5cd889d3979 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -342,10 +342,24 @@ def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]: # at the base we have any global configuration yield kinds.GLOBAL, config_files[kinds.GLOBAL] + site_accessable = int + site_index = 0 + site_all_accessable = bool + + for fname in config_files[kinds.SITE]: + site_index += 1 + if os.access(fname, os.W_OK): + site_accessable += 1 + + if site_accessable < site_index: + site_all_accessable = False + elif site_accessable == site_index: + site_all_accessable = True + # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) or not os.access(config_files[kinds.SITE], os.W_OK) + ) or not site_all_accessable == True if should_load_user_config: # The legacy config file is overridden by the new config file From 54be97e05ce6e303dddb95f9d8d0291c35f7189c Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Mon, 15 Aug 2022 16:43:58 -0700 Subject: [PATCH 36/70] Use strict optional checking in glibc Suggested by pradyunsg in #11374 `--no-strict-optional` defeats half the purpose of using mypy. This change is trivial, we already catch AttributeError in the case that mypy is concerned about. --- src/pip/_internal/utils/glibc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index 7bd3c20681d..2f64f5aa33d 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -1,6 +1,3 @@ -# The following comment should be removed at some point in the future. -# mypy: strict-optional=False - import os import sys from typing import Optional, Tuple @@ -21,7 +18,7 @@ def glibc_version_string_confstr() -> Optional[str]: return None try: # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = os.confstr("CS_GNU_LIBC_VERSION").split() + _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore[union-attr] except (AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None From b5a40ed64bf534858f9cd43d0a9fcd86e478c836 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:45:24 -0700 Subject: [PATCH 37/70] news --- news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst diff --git a/news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst b/news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst new file mode 100644 index 00000000000..e69de29bb2d From 42117756313b1b59c3a6b4f637795688fc36b19e Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:48:49 -0700 Subject: [PATCH 38/70] remove the error code to silence ruff --- src/pip/_internal/utils/glibc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index 2f64f5aa33d..d0e1dbc2c47 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -18,7 +18,7 @@ def glibc_version_string_confstr() -> Optional[str]: return None try: # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore[union-attr] + _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore except (AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None From 36014e6f495bd57363e935e466da8f165acd51f6 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:51:56 -0700 Subject: [PATCH 39/70] don't catch attributeerror --- src/pip/_internal/utils/glibc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index d0e1dbc2c47..b9dbfbcbf30 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -17,9 +17,12 @@ def glibc_version_string_confstr() -> Optional[str]: if sys.platform == "win32": return None try: + gnu_libc_version = os.confstr("CS_GNU_LIBC_VERSION") + if gnu_libc_version is None: + return None # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore - except (AttributeError, OSError, ValueError): + _, version = gnu_libc_version + except (OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None return version From e995f2564495d0bb0cb609c0b48091c3f5708ed8 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:54:20 -0700 Subject: [PATCH 40/70] nope --- src/pip/_internal/utils/glibc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index b9dbfbcbf30..b80c1881bd3 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -21,7 +21,7 @@ def glibc_version_string_confstr() -> Optional[str]: if gnu_libc_version is None: return None # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = gnu_libc_version + _, version = gnu_libc_version.split() except (OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None From b5377aeb73ac1fabc45abb07ed92b209b3213e98 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 02:04:36 -0700 Subject: [PATCH 41/70] nope --- src/pip/_internal/utils/glibc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index b80c1881bd3..81342afa447 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -22,7 +22,7 @@ def glibc_version_string_confstr() -> Optional[str]: return None # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": _, version = gnu_libc_version.split() - except (OSError, ValueError): + except (AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None return version From c57bad63da242068f74e1eb96c97c3bc800b7f88 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 22 Jun 2023 11:27:48 +0800 Subject: [PATCH 42/70] Add permission check before configuration10 --- src/pip/_internal/configuration.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 5cd889d3979..a44f13321a0 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,12 +211,12 @@ def save(self) -> None: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - if os.access(fname, os.W_OK): + try: with open(fname, "w") as f: parser.write(f) - else: + except IOError as error: raise ConfigurationError( - "Configuation file not writeable {}".format(': '.join(fname)) + "An error occurred while writing to the configuration file: {0}\nError message: {1}".format(fname, error) ) # @@ -342,24 +342,10 @@ def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]: # at the base we have any global configuration yield kinds.GLOBAL, config_files[kinds.GLOBAL] - site_accessable = int - site_index = 0 - site_all_accessable = bool - - for fname in config_files[kinds.SITE]: - site_index += 1 - if os.access(fname, os.W_OK): - site_accessable += 1 - - if site_accessable < site_index: - site_all_accessable = False - elif site_accessable == site_index: - site_all_accessable = True - # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) or not site_all_accessable == True + ) if should_load_user_config: # The legacy config file is overridden by the new config file From c4709d2b2251528647e19be0d0f8ea83d1011e24 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 22 Jun 2023 11:31:43 +0800 Subject: [PATCH 43/70] Add permission check before configuration11 --- src/pip/_internal/configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index a44f13321a0..869842e3742 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,7 +216,8 @@ def save(self) -> None: parser.write(f) except IOError as error: raise ConfigurationError( - "An error occurred while writing to the configuration file: {0}\nError message: {1}".format(fname, error) + "An error occurred while writing to the configuration file: {0}\n \ + Error message: {1}".format(fname, error) ) # From 7572dbc09581c139199496adc57fefb9f404a6b2 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 22 Jun 2023 12:03:58 +0800 Subject: [PATCH 44/70] Add IO check before save configuration1 --- news/11920.bugfix.rst | 2 +- src/pip/_internal/configuration.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/news/11920.bugfix.rst b/news/11920.bugfix.rst index f91667c5251..d8e22ee9bd7 100644 --- a/news/11920.bugfix.rst +++ b/news/11920.bugfix.rst @@ -1 +1 @@ -Add permission check before configuration \ No newline at end of file +Add permission check before configuration diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 869842e3742..46562652faa 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,8 +216,10 @@ def save(self) -> None: parser.write(f) except IOError as error: raise ConfigurationError( - "An error occurred while writing to the configuration file: {0}\n \ - Error message: {1}".format(fname, error) + "An error occurred while writing to the configuration file: {0}\n \ + Error message: {1}".format( + fname, error + ) ) # @@ -346,7 +348,7 @@ def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]: # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) + ) if should_load_user_config: # The legacy config file is overridden by the new config file From 9fa64244522a237725e95e36eaedcef4ac25e87c Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 25 Jun 2023 12:06:02 +0100 Subject: [PATCH 45/70] Remove the no-response workflow (#12102) --- .github/workflows/no-response.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .github/workflows/no-response.yml diff --git a/.github/workflows/no-response.yml b/.github/workflows/no-response.yml deleted file mode 100644 index 939290b93e5..00000000000 --- a/.github/workflows/no-response.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: No Response - -# Both `issue_comment` and `scheduled` event types are required for this Action -# to work properly. -on: - issue_comment: - types: [created] - schedule: - # Schedule for five minutes after the hour, every hour - - cron: '5 * * * *' - -jobs: - noResponse: - runs-on: ubuntu-latest - steps: - - uses: lee-dohm/no-response@v0.5.0 - with: - token: ${{ github.token }} - responseRequiredLabel: "S: awaiting response" From 108c055f727e91b366b3f12854d1dccc39195d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 27 Jun 2023 15:10:38 +0200 Subject: [PATCH 46/70] Revert "xfail test_pip_wheel_ext_module_with_tmpdir_inside" This reverts commit fab519dfd936def55ed019a0da18d0d77509fb95. --- tests/functional/test_wheel.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index cfaef541dcf..c0e27949256 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -343,15 +343,6 @@ def test_pip_wheel_with_user_set_in_config( sys.platform.startswith("win"), reason="The empty extension module does not work on Win", ) -@pytest.mark.xfail( - condition=sys.platform == "darwin" and sys.version_info < (3, 9), - reason=( - "Unexplained 'no module named platform' in " - "https://github.com/pypa/wheel/blob" - "/c87e6ed82b58b41b258a3e8c852af8bc1817bb00" - "/src/wheel/vendored/packaging/tags.py#L396-L411" - ), -) def test_pip_wheel_ext_module_with_tmpdir_inside( script: PipTestEnvironment, data: TestData, common_wheels: Path ) -> None: From a0976d8832f52c2f14472f7b20b1cf1776a63ac8 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 27 Jun 2023 14:47:09 +0100 Subject: [PATCH 47/70] Fix lint issues --- tests/unit/test_network_auth.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 5dde6da57c5..e3cb772bb05 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -193,7 +193,7 @@ def test_keyring_get_password( expect: Tuple[Optional[str], Optional[str]], ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path2", "http://example.com/path3"], keyring_provider="import", @@ -205,7 +205,7 @@ def test_keyring_get_password( def test_keyring_get_password_after_prompt(monkeypatch: pytest.MonkeyPatch) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -221,7 +221,7 @@ def test_keyring_get_password_after_prompt_when_none( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -242,7 +242,7 @@ def test_keyring_get_password_username_in_index( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://user@example.com/path2", "http://example.com/path4"], keyring_provider="import", @@ -278,7 +278,7 @@ def test_keyring_set_password( expect_save: bool, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(prompting=True, keyring_provider="import") monkeypatch.setattr(auth, "_get_url_and_credentials", lambda u: (u, None, None)) monkeypatch.setattr(auth, "_prompt_for_password", lambda *a: creds) @@ -354,7 +354,7 @@ def get_credential(self, system: str, username: str) -> Optional[Credential]: def test_keyring_get_credential( monkeypatch: pytest.MonkeyPatch, url: str, expect: str ) -> None: - monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path1", "http://example.com/path2"], keyring_provider="import", @@ -378,7 +378,7 @@ def get_credential(self, system: str, username: str) -> None: def test_broken_keyring_disables_keyring(monkeypatch: pytest.MonkeyPatch) -> None: keyring_broken = KeyringModuleBroken() - monkeypatch.setitem(sys.modules, "keyring", keyring_broken) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring_broken) auth = MultiDomainBasicAuth( index_urls=["http://example.com/"], keyring_provider="import" From c7daa07f6a65c73173f623c1be34ed2956628715 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 27 Jun 2023 14:47:39 +0100 Subject: [PATCH 48/70] Reword the check for no hashes --- src/pip/_internal/models/link.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 3cfc3e8c4fe..4453519ad02 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -112,9 +112,9 @@ def supported_hashes(hashes: Optional[Dict[str, str]]) -> Optional[Dict[str, str if hashes is None: return None hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} - if len(hashes) > 0: - return hashes - return None + if not hashes: + return None + return hashes def _clean_url_path_part(part: str) -> str: From fab8cf7479f573a9284ae4c9a85f776c951c6656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 27 Jun 2023 15:49:40 +0200 Subject: [PATCH 49/70] Remove Unused "type: ignore" comments --- tests/unit/test_network_auth.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 5dde6da57c5..e3cb772bb05 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -193,7 +193,7 @@ def test_keyring_get_password( expect: Tuple[Optional[str], Optional[str]], ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path2", "http://example.com/path3"], keyring_provider="import", @@ -205,7 +205,7 @@ def test_keyring_get_password( def test_keyring_get_password_after_prompt(monkeypatch: pytest.MonkeyPatch) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -221,7 +221,7 @@ def test_keyring_get_password_after_prompt_when_none( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -242,7 +242,7 @@ def test_keyring_get_password_username_in_index( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://user@example.com/path2", "http://example.com/path4"], keyring_provider="import", @@ -278,7 +278,7 @@ def test_keyring_set_password( expect_save: bool, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(prompting=True, keyring_provider="import") monkeypatch.setattr(auth, "_get_url_and_credentials", lambda u: (u, None, None)) monkeypatch.setattr(auth, "_prompt_for_password", lambda *a: creds) @@ -354,7 +354,7 @@ def get_credential(self, system: str, username: str) -> Optional[Credential]: def test_keyring_get_credential( monkeypatch: pytest.MonkeyPatch, url: str, expect: str ) -> None: - monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path1", "http://example.com/path2"], keyring_provider="import", @@ -378,7 +378,7 @@ def get_credential(self, system: str, username: str) -> None: def test_broken_keyring_disables_keyring(monkeypatch: pytest.MonkeyPatch) -> None: keyring_broken = KeyringModuleBroken() - monkeypatch.setitem(sys.modules, "keyring", keyring_broken) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring_broken) auth = MultiDomainBasicAuth( index_urls=["http://example.com/"], keyring_provider="import" From 782cff7e0121d5160acddbfae2ef41e98492ffe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 10 Apr 2023 15:59:59 +0200 Subject: [PATCH 50/70] Warn when legacy versions and specifiers are resolved Also warn in pip check. ... --- news/12063.removal.rst | 2 ++ src/pip/_internal/commands/check.py | 2 ++ src/pip/_internal/commands/download.py | 1 + src/pip/_internal/commands/install.py | 3 ++ src/pip/_internal/commands/wheel.py | 1 + src/pip/_internal/operations/check.py | 38 ++++++++++++++++++++++++++ src/pip/_internal/req/req_set.py | 37 +++++++++++++++++++++++++ 7 files changed, 84 insertions(+) create mode 100644 news/12063.removal.rst diff --git a/news/12063.removal.rst b/news/12063.removal.rst new file mode 100644 index 00000000000..037b0c6089a --- /dev/null +++ b/news/12063.removal.rst @@ -0,0 +1,2 @@ +Deprecate legacy version and version specifiers that don't conform to `PEP 440 +`_ diff --git a/src/pip/_internal/commands/check.py b/src/pip/_internal/commands/check.py index 584df9f55c5..5efd0a34160 100644 --- a/src/pip/_internal/commands/check.py +++ b/src/pip/_internal/commands/check.py @@ -7,6 +7,7 @@ from pip._internal.operations.check import ( check_package_set, create_package_set_from_installed, + warn_legacy_versions_and_specifiers, ) from pip._internal.utils.misc import write_output @@ -21,6 +22,7 @@ class CheckCommand(Command): def run(self, options: Values, args: List[str]) -> int: package_set, parsing_probs = create_package_set_from_installed() + warn_legacy_versions_and_specifiers(package_set) missing, conflicting = check_package_set(package_set) for project_name in missing: diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 36e947c8c05..63bd53a50c8 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -130,6 +130,7 @@ def run(self, options: Values, args: List[str]) -> int: self.trace_basic_info(finder) requirement_set = resolver.resolve(reqs, check_supported_wheels=True) + requirement_set.warn_legacy_versions_and_specifiers() downloaded: List[str] = [] for req in requirement_set.requirements.values(): diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 3c15ed4158c..f6a300804f4 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -387,6 +387,9 @@ def run(self, options: Values, args: List[str]) -> int: json.dump(report.to_dict(), f, indent=2, ensure_ascii=False) if options.dry_run: + # In non dry-run mode, the legacy versions and specifiers check + # will be done as part of conflict detection. + requirement_set.warn_legacy_versions_and_specifiers() would_install_items = sorted( (r.metadata["name"], r.metadata["version"]) for r in requirement_set.requirements_to_install diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index c6a588ff09b..e6735bd8da7 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -145,6 +145,7 @@ def run(self, options: Values, args: List[str]) -> int: self.trace_basic_info(finder) requirement_set = resolver.resolve(reqs, check_supported_wheels=True) + requirement_set.warn_legacy_versions_and_specifiers() reqs_to_build: List[InstallRequirement] = [] for req in requirement_set.requirements.values(): diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index e3bce69b204..2610459228f 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -5,12 +5,15 @@ from typing import Callable, Dict, List, NamedTuple, Optional, Set, Tuple from pip._vendor.packaging.requirements import Requirement +from pip._vendor.packaging.specifiers import LegacySpecifier from pip._vendor.packaging.utils import NormalizedName, canonicalize_name +from pip._vendor.packaging.version import LegacyVersion from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.metadata import get_default_environment from pip._internal.metadata.base import DistributionVersion from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.deprecation import deprecated logger = logging.getLogger(__name__) @@ -57,6 +60,8 @@ def check_package_set( package name and returns a boolean. """ + warn_legacy_versions_and_specifiers(package_set) + missing = {} conflicting = {} @@ -147,3 +152,36 @@ def _create_whitelist( break return packages_affected + + +def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None: + for project_name, package_details in package_set.items(): + if isinstance(package_details.version, LegacyVersion): + deprecated( + reason=( + f"{project_name} {package_details.version} " + f"has a non-standard version number." + ), + replacement=( + f"to upgrade to a newer version of {project_name} " + f"or contact the author to suggest that they " + f"release a version with a conforming version number" + ), + issue=12063, + gone_in="23.3", + ) + for dep in package_details.dependencies: + if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier): + deprecated( + reason=( + f"{project_name} {package_details.version} " + f"has a non-standard dependency specifier {dep}." + ), + replacement=( + f"to upgrade to a newer version of {project_name} " + f"or contact the author to suggest that they " + f"release a version with a conforming dependency specifiers" + ), + issue=12063, + gone_in="23.3", + ) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index ec7a6e07a25..cff67601737 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -2,9 +2,12 @@ from collections import OrderedDict from typing import Dict, List +from pip._vendor.packaging.specifiers import LegacySpecifier from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.packaging.version import LegacyVersion from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.deprecation import deprecated logger = logging.getLogger(__name__) @@ -80,3 +83,37 @@ def requirements_to_install(self) -> List[InstallRequirement]: for install_req in self.all_requirements if not install_req.constraint and not install_req.satisfied_by ] + + def warn_legacy_versions_and_specifiers(self) -> None: + for req in self.requirements_to_install: + version = req.get_dist().version + if isinstance(version, LegacyVersion): + deprecated( + reason=( + f"pip has selected the non standard version {version} " + f"of {req}. In the future this version will be " + f"ignored as it isn't standard compliant." + ), + replacement=( + "set or update constraints to select another version " + "or contact the package author to fix the version number" + ), + issue=12063, + gone_in="23.3", + ) + for dep in req.get_dist().iter_dependencies(): + if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier): + deprecated( + reason=( + f"pip has selected {req} {version} which has non " + f"standard dependency specifier {dep}. " + f"In the future this version of {req} will be " + f"ignored as it isn't standard compliant." + ), + replacement=( + "set or update constraints to select another version " + "or contact the package author to fix the version number" + ), + issue=12063, + gone_in="23.3", + ) From 6507734aac0f3c0972aef5097b8d0b4defb791f8 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 29 Jun 2023 16:51:11 +0800 Subject: [PATCH 51/70] Fix string formatting --- src/pip/_internal/configuration.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 46562652faa..35189d0d2c8 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,10 +216,8 @@ def save(self) -> None: parser.write(f) except IOError as error: raise ConfigurationError( - "An error occurred while writing to the configuration file: {0}\n \ - Error message: {1}".format( - fname, error - ) + f"An error occurred while writing to the configuration file " + f"{fname}: {error}" ) # From 41f138e43a6c54804b7c7fe3d4a8477508ef4a97 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 29 Jun 2023 16:51:34 +0800 Subject: [PATCH 52/70] Minimize changeset --- src/pip/_internal/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 35189d0d2c8..0ed33ac9c75 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -347,7 +347,6 @@ def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]: should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) ) - if should_load_user_config: # The legacy config file is overridden by the new config file yield kinds.USER, config_files[kinds.USER] From 256af8f6912799ebf66b2ebc6707aae2e0487fe1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 29 Jun 2023 17:28:16 +0800 Subject: [PATCH 53/70] Catch OSError instead of IOError --- src/pip/_internal/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 0ed33ac9c75..96f824955bf 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -214,7 +214,7 @@ def save(self) -> None: try: with open(fname, "w") as f: parser.write(f) - except IOError as error: + except OSError as error: raise ConfigurationError( f"An error occurred while writing to the configuration file " f"{fname}: {error}" From 5d0c2773b8bf0cc569c82e6d3697fe7b89c71192 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 30 Jun 2023 11:38:12 +0100 Subject: [PATCH 54/70] Stop using a RAM disk for the Windows tests --- .github/workflows/ci.yml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b35e93b21f..1361980565d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,24 +167,13 @@ jobs: with: python-version: ${{ matrix.python }} - # We use a RAMDisk on Windows, since filesystem IO is a big slowdown - # for our tests. - - name: Create a RAMDisk - run: ./tools/ci/New-RAMDisk.ps1 -Drive R -Size 1GB - - - name: Setup RAMDisk permissions - run: | - mkdir R:\Temp - $acl = Get-Acl "R:\Temp" - $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( - "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow" - ) - $acl.AddAccessRule($rule) - Set-Acl "R:\Temp" $acl - + # We use C:\Temp (which is already available on the worker) + # as a temporary directory for all of the tests because the + # default value (under the user dir) is more deeply nested + # and causes tests to fail with "path too long" errors. - run: pip install nox env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" # Main check - name: Run unit tests @@ -194,7 +183,7 @@ jobs: -m unit --verbose --numprocesses auto --showlocals env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" - name: Run integration tests (group 1) if: matrix.group == 1 @@ -203,7 +192,7 @@ jobs: -m integration -k "not test_install" --verbose --numprocesses auto --showlocals env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" - name: Run integration tests (group 2) if: matrix.group == 2 @@ -212,7 +201,7 @@ jobs: -m integration -k "test_install" --verbose --numprocesses auto --showlocals env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" tests-zipapp: name: tests / zipapp From 45468f06d429080a9042909b76cfc25fce9bee5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 29 May 2023 13:57:52 +0200 Subject: [PATCH 55/70] Pass revisions options explicitly to mercurial commands --- news/12119.bugfix.rst | 3 +++ src/pip/_internal/vcs/mercurial.py | 2 +- tests/unit/test_vcs.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 news/12119.bugfix.rst diff --git a/news/12119.bugfix.rst b/news/12119.bugfix.rst new file mode 100644 index 00000000000..da8d8b04dcd --- /dev/null +++ b/news/12119.bugfix.rst @@ -0,0 +1,3 @@ +Pass the ``-r`` flag to mercurial to be explicit that a revision is passed and protect +against ``hg`` options injection as part of VCS URLs. Users that do not have control on +VCS URLs passed to pip are advised to upgrade. diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 2a005e0aff2..4595960b5bf 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -31,7 +31,7 @@ class Mercurial(VersionControl): @staticmethod def get_base_rev_args(rev: str) -> List[str]: - return [rev] + return ["-r", rev] def fetch_new( self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 566c88cf02b..38daaa0f21d 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -66,7 +66,7 @@ def test_rev_options_repr() -> None: # First check VCS-specific RevOptions behavior. (Bazaar, [], ["-r", "123"], {}), (Git, ["HEAD"], ["123"], {}), - (Mercurial, [], ["123"], {}), + (Mercurial, [], ["-r", "123"], {}), (Subversion, [], ["-r", "123"], {}), # Test extra_args. For this, test using a single VersionControl class. ( From b99e082b003788f2e8abbad47d461f495faad892 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 1 Jul 2023 14:42:10 +0100 Subject: [PATCH 56/70] Record download of completed partial requirements --- src/pip/_internal/operations/prepare.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 130b9737742..8d7151353f0 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -471,6 +471,7 @@ def _complete_partial_requirements( logger.debug("Downloading link %s to %s", link, filepath) req = links_to_fully_download[link] req.local_file_path = filepath + self._downloaded[req.link.url] = filepath # This step is necessary to ensure all lazy wheels are processed # successfully by the 'download', 'wheel', and 'install' commands. From cb25bf3731d46697586fc72a24ba1f8e57311377 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 1 Jul 2023 14:51:10 +0100 Subject: [PATCH 57/70] Add a news file --- news/11847.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11847.bugfix.rst diff --git a/news/11847.bugfix.rst b/news/11847.bugfix.rst new file mode 100644 index 00000000000..1cad477eaa2 --- /dev/null +++ b/news/11847.bugfix.rst @@ -0,0 +1 @@ +Prevent downloading files twice when PEP 658 metadata is present From 647ba8d07e7832ea69d93f9a686d8f276e669a14 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 3 Jul 2023 10:35:01 +0100 Subject: [PATCH 58/70] Limit the double download fix to wheels --- src/pip/_internal/operations/prepare.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 8d7151353f0..5d9bedc031a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -471,7 +471,19 @@ def _complete_partial_requirements( logger.debug("Downloading link %s to %s", link, filepath) req = links_to_fully_download[link] req.local_file_path = filepath - self._downloaded[req.link.url] = filepath + # TODO: This needs fixing for sdists + # This is an emergency fix for #11847, which reports that + # distributions get downloaded twice when metadata is loaded + # from a PEP 658 standalone metadata file. Setting _downloaded + # fixes this for wheels, but breaks the sdist case (tests + # test_download_metadata). As PyPI is currently not serving + # metadata for wheels, this is not an immediate issue. + # Fixing the problem properly looks like it will require a + # complete refactoring of the `prepare_linked_requirements_more` + # logic, and I haven't a clue where to start on that, so for now + # I have fixed the issue *just* for wheels. + if req.is_wheel: + self._downloaded[req.link.url] = filepath # This step is necessary to ensure all lazy wheels are processed # successfully by the 'download', 'wheel', and 'install' commands. From 8e80a3ad9a5b80de72efad6cbad3bebf2328642b Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 3 Jul 2023 10:45:01 +0100 Subject: [PATCH 59/70] Fix typo --- src/pip/_internal/operations/prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 5d9bedc031a..49d86268a3b 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -476,7 +476,7 @@ def _complete_partial_requirements( # distributions get downloaded twice when metadata is loaded # from a PEP 658 standalone metadata file. Setting _downloaded # fixes this for wheels, but breaks the sdist case (tests - # test_download_metadata). As PyPI is currently not serving + # test_download_metadata). As PyPI is currently only serving # metadata for wheels, this is not an immediate issue. # Fixing the problem properly looks like it will require a # complete refactoring of the `prepare_linked_requirements_more` From 5dc65eabb75f89d4f4749b6c764042c227f6870a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD=20=D0=94=D0=BE=D0=BD=D1=87?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Mon, 15 May 2023 22:42:51 +0400 Subject: [PATCH 60/70] Don't exclude setuptools, distribute & wheel from freeze output on Python 3.12+ Due to the advent of build isolation, it is no longer necessary to install setuptools and wheel in an environment just to install other packages. Moreover, on Python 3.12 both ensurepip [1] and virtualenv [2] are to stop installing setuptools & wheel by default. This means that when those packages are present in a Python 3.12+ environment, it is reasonable to assume that they are runtime dependencies of the user's project, and therefore should be included in freeze output. distribute is just obsolete. [1] https://github.com/python/cpython/issues/95299 [2] https://github.com/pypa/virtualenv/pull/2558 --- news/4256.removal.rst | 3 +++ src/pip/_internal/commands/freeze.py | 5 ++++- tests/functional/test_freeze.py | 20 +++++++++++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 news/4256.removal.rst diff --git a/news/4256.removal.rst b/news/4256.removal.rst new file mode 100644 index 00000000000..5440f532add --- /dev/null +++ b/news/4256.removal.rst @@ -0,0 +1,3 @@ +``freeze`` no longer excludes the ``setuptools``, ``distribute`` and ``wheel`` +packages from the output by default when running on Python 3.12 or later. +Use ``--exclude`` if you wish to exclude any of these packages. diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py index 5fa6d39b2c7..87f281d76fb 100644 --- a/src/pip/_internal/commands/freeze.py +++ b/src/pip/_internal/commands/freeze.py @@ -8,7 +8,10 @@ from pip._internal.operations.freeze import freeze from pip._internal.utils.compat import stdlib_pkgs -DEV_PKGS = {"pip", "setuptools", "distribute", "wheel"} +DEV_PKGS = {"pip"} + +if sys.version_info < (3, 12): + DEV_PKGS |= {"setuptools", "distribute", "wheel"} class FreezeCommand(Command): diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index b24b27edcc6..81a660ab6f4 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -88,11 +88,29 @@ def test_basic_freeze(script: PipTestEnvironment) -> None: def test_freeze_with_pip(script: PipTestEnvironment) -> None: - """Test pip shows itself""" + """Test that pip shows itself only when --all is used""" + result = script.pip("freeze") + assert "pip==" not in result.stdout result = script.pip("freeze", "--all") assert "pip==" in result.stdout +def test_freeze_with_setuptools(script: PipTestEnvironment) -> None: + """ + Test that pip shows setuptools only when --all is used + or Python version is >=3.12 + """ + + result = script.pip("freeze") + if sys.version_info >= (3, 12): + assert "setuptools==" in result.stdout + else: + assert "setuptools==" not in result.stdout + + result = script.pip("freeze", "--all") + assert "setuptools==" in result.stdout + + def test_exclude_and_normalization(script: PipTestEnvironment, tmpdir: Path) -> None: req_path = wheel.make_wheel(name="Normalizable_Name", version="1.0").save_to_dir( tmpdir From 393ccfbc31eccdf7f053ee4d62b055e515ef3183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD=20=D0=94=D0=BE=D0=BD=D1=87?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Mon, 29 May 2023 02:23:01 +0400 Subject: [PATCH 61/70] test_freeze_with_setuptools: use mocks This makes it possible to test both branches on any Python version. --- src/pip/_internal/commands/freeze.py | 20 +++++++++++----- tests/functional/test_freeze.py | 34 ++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py index 87f281d76fb..fd9d88a8b01 100644 --- a/src/pip/_internal/commands/freeze.py +++ b/src/pip/_internal/commands/freeze.py @@ -1,6 +1,6 @@ import sys from optparse import Values -from typing import List +from typing import AbstractSet, List from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command @@ -8,10 +8,18 @@ from pip._internal.operations.freeze import freeze from pip._internal.utils.compat import stdlib_pkgs -DEV_PKGS = {"pip"} -if sys.version_info < (3, 12): - DEV_PKGS |= {"setuptools", "distribute", "wheel"} +def _should_suppress_build_backends() -> bool: + return sys.version_info < (3, 12) + + +def _dev_pkgs() -> AbstractSet[str]: + pkgs = {"pip"} + + if _should_suppress_build_backends(): + pkgs |= {"setuptools", "distribute", "wheel"} + + return pkgs class FreezeCommand(Command): @@ -64,7 +72,7 @@ def add_options(self) -> None: action="store_true", help=( "Do not skip these packages in the output:" - " {}".format(", ".join(DEV_PKGS)) + " {}".format(", ".join(_dev_pkgs())) ), ) self.cmd_opts.add_option( @@ -80,7 +88,7 @@ def add_options(self) -> None: def run(self, options: Values, args: List[str]) -> int: skip = set(stdlib_pkgs) if not options.freeze_all: - skip.update(DEV_PKGS) + skip.update(_dev_pkgs()) if options.excludes: skip.update(options.excludes) diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 81a660ab6f4..d6122308a69 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -98,18 +98,38 @@ def test_freeze_with_pip(script: PipTestEnvironment) -> None: def test_freeze_with_setuptools(script: PipTestEnvironment) -> None: """ Test that pip shows setuptools only when --all is used - or Python version is >=3.12 + or _should_suppress_build_backends() returns false """ - result = script.pip("freeze") - if sys.version_info >= (3, 12): - assert "setuptools==" in result.stdout - else: - assert "setuptools==" not in result.stdout - result = script.pip("freeze", "--all") assert "setuptools==" in result.stdout + (script.site_packages_path / "mock.pth").write_text("import mock\n") + + (script.site_packages_path / "mock.py").write_text( + textwrap.dedent( + """\ + import pip._internal.commands.freeze as freeze + freeze._should_suppress_build_backends = lambda: False + """ + ) + ) + + result = script.pip("freeze") + assert "setuptools==" in result.stdout + + (script.site_packages_path / "mock.py").write_text( + textwrap.dedent( + """\ + import pip._internal.commands.freeze as freeze + freeze._should_suppress_build_backends = lambda: True + """ + ) + ) + + result = script.pip("freeze") + assert "setuptools==" not in result.stdout + def test_exclude_and_normalization(script: PipTestEnvironment, tmpdir: Path) -> None: req_path = wheel.make_wheel(name="Normalizable_Name", version="1.0").save_to_dir( From 7a69c00720fb8e660ef0d1df174b79e039bdba95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD=20=D0=94=D0=BE=D0=BD=D1=87?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Fri, 7 Jul 2023 01:48:03 +0300 Subject: [PATCH 62/70] Make the changelog entry more verbose Co-authored-by: Tzu-ping Chung --- news/4256.removal.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/news/4256.removal.rst b/news/4256.removal.rst index 5440f532add..eb89898501b 100644 --- a/news/4256.removal.rst +++ b/news/4256.removal.rst @@ -1,3 +1,4 @@ -``freeze`` no longer excludes the ``setuptools``, ``distribute`` and ``wheel`` -packages from the output by default when running on Python 3.12 or later. -Use ``--exclude`` if you wish to exclude any of these packages. +``freeze`` no longer excludes the ``setuptools``, ``distribute``, and ``wheel`` +from the output when running on Python 3.12 or later, where they are not +included in a virtual environment by default. Use ``--exclude`` if you wish to +exclude any of these packages. From 856c7ec27e8c1dc58a32191c1bcbbc151727a893 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 9 Jul 2023 21:30:08 +0100 Subject: [PATCH 63/70] Upgrade platformdirs to 3.8.1 --- news/platformdirs.vendor.rst | 1 + src/pip/_vendor/platformdirs/__init__.py | 143 +++++++++++++++-------- src/pip/_vendor/platformdirs/__main__.py | 24 ++-- src/pip/_vendor/platformdirs/android.py | 112 +++++++++++++++--- src/pip/_vendor/platformdirs/api.py | 70 ++++++++--- src/pip/_vendor/platformdirs/macos.py | 33 +++++- src/pip/_vendor/platformdirs/unix.py | 97 +++++++++------ src/pip/_vendor/platformdirs/version.py | 4 +- src/pip/_vendor/platformdirs/windows.py | 104 +++++++++++++---- src/pip/_vendor/vendor.txt | 2 +- 10 files changed, 441 insertions(+), 149 deletions(-) create mode 100644 news/platformdirs.vendor.rst diff --git a/news/platformdirs.vendor.rst b/news/platformdirs.vendor.rst new file mode 100644 index 00000000000..f396d84a666 --- /dev/null +++ b/news/platformdirs.vendor.rst @@ -0,0 +1 @@ +Upgrade platformdirs to 3.8.1 diff --git a/src/pip/_vendor/platformdirs/__init__.py b/src/pip/_vendor/platformdirs/__init__.py index c46a145cdc1..5ebf5957b46 100644 --- a/src/pip/_vendor/platformdirs/__init__.py +++ b/src/pip/_vendor/platformdirs/__init__.py @@ -6,17 +6,20 @@ import os import sys -from pathlib import Path - -if sys.version_info >= (3, 8): # pragma: no cover (py38+) - from typing import Literal -else: # pragma: no cover (py38+) - from pip._vendor.typing_extensions import Literal +from typing import TYPE_CHECKING from .api import PlatformDirsABC from .version import __version__ from .version import __version_tuple__ as __version_info__ +if TYPE_CHECKING: + from pathlib import Path + + if sys.version_info >= (3, 8): # pragma: no cover (py38+) + from typing import Literal + else: # pragma: no cover (py38+) + from pip._vendor.typing_extensions import Literal + def _set_platform_dir_class() -> type[PlatformDirsABC]: if sys.platform == "win32": @@ -48,8 +51,8 @@ def user_data_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -72,8 +75,8 @@ def site_data_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -96,8 +99,8 @@ def user_config_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -120,8 +123,8 @@ def site_config_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -144,8 +147,8 @@ def user_cache_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -168,8 +171,8 @@ def site_cache_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -192,8 +195,8 @@ def user_state_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -216,8 +219,8 @@ def user_log_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -237,18 +240,36 @@ def user_log_dir( def user_documents_dir() -> str: - """ - :returns: documents directory tied to the user - """ + """:returns: documents directory tied to the user""" return PlatformDirs().user_documents_dir +def user_downloads_dir() -> str: + """:returns: downloads directory tied to the user""" + return PlatformDirs().user_downloads_dir + + +def user_pictures_dir() -> str: + """:returns: pictures directory tied to the user""" + return PlatformDirs().user_pictures_dir + + +def user_videos_dir() -> str: + """:returns: videos directory tied to the user""" + return PlatformDirs().user_videos_dir + + +def user_music_dir() -> str: + """:returns: music directory tied to the user""" + return PlatformDirs().user_music_dir + + def user_runtime_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -271,8 +292,8 @@ def user_data_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -295,8 +316,8 @@ def site_data_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -319,8 +340,8 @@ def user_config_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -343,8 +364,8 @@ def site_config_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -367,8 +388,8 @@ def site_cache_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -391,8 +412,8 @@ def user_cache_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -415,8 +436,8 @@ def user_state_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -439,8 +460,8 @@ def user_log_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -460,18 +481,36 @@ def user_log_path( def user_documents_path() -> Path: - """ - :returns: documents path tied to the user - """ + """:returns: documents path tied to the user""" return PlatformDirs().user_documents_path +def user_downloads_path() -> Path: + """:returns: downloads path tied to the user""" + return PlatformDirs().user_downloads_path + + +def user_pictures_path() -> Path: + """:returns: pictures path tied to the user""" + return PlatformDirs().user_pictures_path + + +def user_videos_path() -> Path: + """:returns: videos path tied to the user""" + return PlatformDirs().user_videos_path + + +def user_music_path() -> Path: + """:returns: music path tied to the user""" + return PlatformDirs().user_music_path + + def user_runtime_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -502,6 +541,10 @@ def user_runtime_path( "user_state_dir", "user_log_dir", "user_documents_dir", + "user_downloads_dir", + "user_pictures_dir", + "user_videos_dir", + "user_music_dir", "user_runtime_dir", "site_data_dir", "site_config_dir", @@ -512,6 +555,10 @@ def user_runtime_path( "user_state_path", "user_log_path", "user_documents_path", + "user_downloads_path", + "user_pictures_path", + "user_videos_path", + "user_music_path", "user_runtime_path", "site_data_path", "site_config_path", diff --git a/src/pip/_vendor/platformdirs/__main__.py b/src/pip/_vendor/platformdirs/__main__.py index 7171f13114e..6a0d6dd12e3 100644 --- a/src/pip/_vendor/platformdirs/__main__.py +++ b/src/pip/_vendor/platformdirs/__main__.py @@ -1,3 +1,4 @@ +"""Main entry point.""" from __future__ import annotations from pip._vendor.platformdirs import PlatformDirs, __version__ @@ -9,6 +10,10 @@ "user_state_dir", "user_log_dir", "user_documents_dir", + "user_downloads_dir", + "user_pictures_dir", + "user_videos_dir", + "user_music_dir", "user_runtime_dir", "site_data_dir", "site_config_dir", @@ -17,30 +22,31 @@ def main() -> None: + """Run main entry point.""" app_name = "MyApp" app_author = "MyCompany" - print(f"-- platformdirs {__version__} --") + print(f"-- platformdirs {__version__} --") # noqa: T201 - print("-- app dirs (with optional 'version')") + print("-- app dirs (with optional 'version')") # noqa: T201 dirs = PlatformDirs(app_name, app_author, version="1.0") for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 - print("\n-- app dirs (without optional 'version')") + print("\n-- app dirs (without optional 'version')") # noqa: T201 dirs = PlatformDirs(app_name, app_author) for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 - print("\n-- app dirs (without optional 'appauthor')") + print("\n-- app dirs (without optional 'appauthor')") # noqa: T201 dirs = PlatformDirs(app_name) for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 - print("\n-- app dirs (with disabled 'appauthor')") + print("\n-- app dirs (with disabled 'appauthor')") # noqa: T201 dirs = PlatformDirs(app_name, appauthor=False) for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 if __name__ == "__main__": diff --git a/src/pip/_vendor/platformdirs/android.py b/src/pip/_vendor/platformdirs/android.py index f6de7451b25..76527dda41f 100644 --- a/src/pip/_vendor/platformdirs/android.py +++ b/src/pip/_vendor/platformdirs/android.py @@ -1,3 +1,4 @@ +"""Android.""" from __future__ import annotations import os @@ -30,7 +31,8 @@ def site_data_dir(self) -> str: @property def user_config_dir(self) -> str: """ - :return: config directory tied to the user, e.g. ``/data/user///shared_prefs/`` + :return: config directory tied to the user, e.g. \ + ``/data/user///shared_prefs/`` """ return self._append_app_name_and_version(cast(str, _android_folder()), "shared_prefs") @@ -62,16 +64,34 @@ def user_log_dir(self) -> str: """ path = self.user_cache_dir if self.opinion: - path = os.path.join(path, "log") + path = os.path.join(path, "log") # noqa: PTH118 return path @property def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents`` - """ + """:return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents``""" return _android_documents_folder() + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user e.g. ``/storage/emulated/0/Downloads``""" + return _android_downloads_folder() + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user e.g. ``/storage/emulated/0/Pictures``""" + return _android_pictures_folder() + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user e.g. ``/storage/emulated/0/DCIM/Camera``""" + return _android_videos_folder() + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user e.g. ``/storage/emulated/0/Music``""" + return _android_music_folder() + @property def user_runtime_dir(self) -> str: """ @@ -80,20 +100,20 @@ def user_runtime_dir(self) -> str: """ path = self.user_cache_dir if self.opinion: - path = os.path.join(path, "tmp") + path = os.path.join(path, "tmp") # noqa: PTH118 return path @lru_cache(maxsize=1) def _android_folder() -> str | None: - """:return: base folder for the Android OS or None if cannot be found""" + """:return: base folder for the Android OS or None if it cannot be found""" try: # First try to get path to android app via pyjnius from jnius import autoclass - Context = autoclass("android.content.Context") # noqa: N806 - result: str | None = Context.getFilesDir().getParentFile().getAbsolutePath() - except Exception: + context = autoclass("android.content.Context") + result: str | None = context.getFilesDir().getParentFile().getAbsolutePath() + except Exception: # noqa: BLE001 # if fails find an android folder looking path on the sys.path pattern = re.compile(r"/data/(data|user/\d+)/(.+)/files") for path in sys.path: @@ -112,15 +132,79 @@ def _android_documents_folder() -> str: try: from jnius import autoclass - Context = autoclass("android.content.Context") # noqa: N806 - Environment = autoclass("android.os.Environment") # noqa: N806 - documents_dir: str = Context.getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath() - except Exception: + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + documents_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DOCUMENTS).getAbsolutePath() + except Exception: # noqa: BLE001 documents_dir = "/storage/emulated/0/Documents" return documents_dir +@lru_cache(maxsize=1) +def _android_downloads_folder() -> str: + """:return: downloads folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + downloads_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DOWNLOADS).getAbsolutePath() + except Exception: # noqa: BLE001 + downloads_dir = "/storage/emulated/0/Downloads" + + return downloads_dir + + +@lru_cache(maxsize=1) +def _android_pictures_folder() -> str: + """:return: pictures folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + pictures_dir: str = context.getExternalFilesDir(environment.DIRECTORY_PICTURES).getAbsolutePath() + except Exception: # noqa: BLE001 + pictures_dir = "/storage/emulated/0/Pictures" + + return pictures_dir + + +@lru_cache(maxsize=1) +def _android_videos_folder() -> str: + """:return: videos folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + videos_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DCIM).getAbsolutePath() + except Exception: # noqa: BLE001 + videos_dir = "/storage/emulated/0/DCIM/Camera" + + return videos_dir + + +@lru_cache(maxsize=1) +def _android_music_folder() -> str: + """:return: music folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + music_dir: str = context.getExternalFilesDir(environment.DIRECTORY_MUSIC).getAbsolutePath() + except Exception: # noqa: BLE001 + music_dir = "/storage/emulated/0/Music" + + return music_dir + + __all__ = [ "Android", ] diff --git a/src/pip/_vendor/platformdirs/api.py b/src/pip/_vendor/platformdirs/api.py index f140e8b6db8..d64ebb9d45c 100644 --- a/src/pip/_vendor/platformdirs/api.py +++ b/src/pip/_vendor/platformdirs/api.py @@ -1,29 +1,33 @@ +"""Base API.""" from __future__ import annotations import os -import sys from abc import ABC, abstractmethod from pathlib import Path +from typing import TYPE_CHECKING -if sys.version_info >= (3, 8): # pragma: no branch - from typing import Literal # pragma: no cover +if TYPE_CHECKING: + import sys + + if sys.version_info >= (3, 8): # pragma: no cover (py38+) + from typing import Literal + else: # pragma: no cover (py38+) + from pip._vendor.typing_extensions import Literal class PlatformDirsABC(ABC): - """ - Abstract base class for platform directories. - """ + """Abstract base class for platform directories.""" - def __init__( + def __init__( # noqa: PLR0913 self, appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - multipath: bool = False, - opinion: bool = True, - ensure_exists: bool = False, - ): + roaming: bool = False, # noqa: FBT001, FBT002 + multipath: bool = False, # noqa: FBT001, FBT002 + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 + ) -> None: """ Create a new platform directory. @@ -70,7 +74,7 @@ def _append_app_name_and_version(self, *base: str) -> str: params.append(self.appname) if self.version: params.append(self.version) - path = os.path.join(base[0], *params) + path = os.path.join(base[0], *params) # noqa: PTH118 self._optionally_create_directory(path) return path @@ -123,6 +127,26 @@ def user_log_dir(self) -> str: def user_documents_dir(self) -> str: """:return: documents directory tied to the user""" + @property + @abstractmethod + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user""" + + @property + @abstractmethod + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user""" + + @property + @abstractmethod + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user""" + + @property + @abstractmethod + def user_music_dir(self) -> str: + """:return: music directory tied to the user""" + @property @abstractmethod def user_runtime_dir(self) -> str: @@ -173,6 +197,26 @@ def user_documents_path(self) -> Path: """:return: documents path tied to the user""" return Path(self.user_documents_dir) + @property + def user_downloads_path(self) -> Path: + """:return: downloads path tied to the user""" + return Path(self.user_downloads_dir) + + @property + def user_pictures_path(self) -> Path: + """:return: pictures path tied to the user""" + return Path(self.user_pictures_dir) + + @property + def user_videos_path(self) -> Path: + """:return: videos path tied to the user""" + return Path(self.user_videos_dir) + + @property + def user_music_path(self) -> Path: + """:return: music path tied to the user""" + return Path(self.user_music_dir) + @property def user_runtime_path(self) -> Path: """:return: runtime path tied to the user""" diff --git a/src/pip/_vendor/platformdirs/macos.py b/src/pip/_vendor/platformdirs/macos.py index ec9751129c1..a753e2a3aa2 100644 --- a/src/pip/_vendor/platformdirs/macos.py +++ b/src/pip/_vendor/platformdirs/macos.py @@ -1,6 +1,7 @@ +"""macOS.""" from __future__ import annotations -import os +import os.path from .api import PlatformDirsABC @@ -17,7 +18,7 @@ class MacOS(PlatformDirsABC): @property def user_data_dir(self) -> str: """:return: data directory tied to the user, e.g. ``~/Library/Application Support/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support")) # noqa: PTH111 @property def site_data_dir(self) -> str: @@ -37,7 +38,7 @@ def site_config_dir(self) -> str: @property def user_cache_dir(self) -> str: """:return: cache directory tied to the user, e.g. ``~/Library/Caches/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches")) # noqa: PTH111 @property def site_cache_dir(self) -> str: @@ -52,17 +53,37 @@ def user_state_dir(self) -> str: @property def user_log_dir(self) -> str: """:return: log directory tied to the user, e.g. ``~/Library/Logs/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs")) # noqa: PTH111 @property def user_documents_dir(self) -> str: """:return: documents directory tied to the user, e.g. ``~/Documents``""" - return os.path.expanduser("~/Documents") + return os.path.expanduser("~/Documents") # noqa: PTH111 + + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user, e.g. ``~/Downloads``""" + return os.path.expanduser("~/Downloads") # noqa: PTH111 + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user, e.g. ``~/Pictures``""" + return os.path.expanduser("~/Pictures") # noqa: PTH111 + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user, e.g. ``~/Movies``""" + return os.path.expanduser("~/Movies") # noqa: PTH111 + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user, e.g. ``~/Music``""" + return os.path.expanduser("~/Music") # noqa: PTH111 @property def user_runtime_dir(self) -> str: """:return: runtime directory tied to the user, e.g. ``~/Library/Caches/TemporaryItems/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems")) # noqa: PTH111 __all__ = [ diff --git a/src/pip/_vendor/platformdirs/unix.py b/src/pip/_vendor/platformdirs/unix.py index 17d355da9f4..468b0ab4957 100644 --- a/src/pip/_vendor/platformdirs/unix.py +++ b/src/pip/_vendor/platformdirs/unix.py @@ -1,3 +1,4 @@ +"""Unix.""" from __future__ import annotations import os @@ -7,12 +8,14 @@ from .api import PlatformDirsABC -if sys.platform.startswith("linux"): # pragma: no branch # no op check, only to please the type checker - from os import getuid -else: +if sys.platform == "win32": def getuid() -> int: - raise RuntimeError("should only be used on Linux") + msg = "should only be used on Unix" + raise RuntimeError(msg) + +else: + from os import getuid class Unix(PlatformDirsABC): @@ -36,7 +39,7 @@ def user_data_dir(self) -> str: """ path = os.environ.get("XDG_DATA_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.local/share") + path = os.path.expanduser("~/.local/share") # noqa: PTH111 return self._append_app_name_and_version(path) @property @@ -56,7 +59,7 @@ def _with_multi_path(self, path: str) -> str: path_list = path.split(os.pathsep) if not self.multipath: path_list = path_list[0:1] - path_list = [self._append_app_name_and_version(os.path.expanduser(p)) for p in path_list] + path_list = [self._append_app_name_and_version(os.path.expanduser(p)) for p in path_list] # noqa: PTH111 return os.pathsep.join(path_list) @property @@ -67,7 +70,7 @@ def user_config_dir(self) -> str: """ path = os.environ.get("XDG_CONFIG_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.config") + path = os.path.expanduser("~/.config") # noqa: PTH111 return self._append_app_name_and_version(path) @property @@ -91,15 +94,13 @@ def user_cache_dir(self) -> str: """ path = os.environ.get("XDG_CACHE_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.cache") + path = os.path.expanduser("~/.cache") # noqa: PTH111 return self._append_app_name_and_version(path) @property def site_cache_dir(self) -> str: - """ - :return: cache directory shared by users, e.g. ``/var/tmp/$appname/$version`` - """ - return self._append_app_name_and_version("/var/tmp") + """:return: cache directory shared by users, e.g. ``/var/tmp/$appname/$version``""" + return self._append_app_name_and_version("/var/tmp") # noqa: S108 @property def user_state_dir(self) -> str: @@ -109,41 +110,60 @@ def user_state_dir(self) -> str: """ path = os.environ.get("XDG_STATE_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.local/state") + path = os.path.expanduser("~/.local/state") # noqa: PTH111 return self._append_app_name_and_version(path) @property def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_state_dir` if not opinionated else ``log`` in it - """ + """:return: log directory tied to the user, same as `user_state_dir` if not opinionated else ``log`` in it""" path = self.user_state_dir if self.opinion: - path = os.path.join(path, "log") + path = os.path.join(path, "log") # noqa: PTH118 return path @property def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user, e.g. ``~/Documents`` - """ - documents_dir = _get_user_dirs_folder("XDG_DOCUMENTS_DIR") - if documents_dir is None: - documents_dir = os.environ.get("XDG_DOCUMENTS_DIR", "").strip() - if not documents_dir: - documents_dir = os.path.expanduser("~/Documents") + """:return: documents directory tied to the user, e.g. ``~/Documents``""" + return _get_user_media_dir("XDG_DOCUMENTS_DIR", "~/Documents") - return documents_dir + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user, e.g. ``~/Downloads``""" + return _get_user_media_dir("XDG_DOWNLOAD_DIR", "~/Downloads") + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user, e.g. ``~/Pictures``""" + return _get_user_media_dir("XDG_PICTURES_DIR", "~/Pictures") + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user, e.g. ``~/Videos``""" + return _get_user_media_dir("XDG_VIDEOS_DIR", "~/Videos") + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user, e.g. ``~/Music``""" + return _get_user_media_dir("XDG_MUSIC_DIR", "~/Music") @property def user_runtime_dir(self) -> str: """ :return: runtime directory tied to the user, e.g. ``/run/user/$(id -u)/$appname/$version`` or - ``$XDG_RUNTIME_DIR/$appname/$version`` + ``$XDG_RUNTIME_DIR/$appname/$version``. + + For FreeBSD/OpenBSD/NetBSD, it would return ``/var/run/user/$(id -u)/$appname/$version`` if + exists, otherwise ``/tmp/runtime-$(id -u)/$appname/$version``, if``$XDG_RUNTIME_DIR`` + is not set. """ path = os.environ.get("XDG_RUNTIME_DIR", "") if not path.strip(): - path = f"/run/user/{getuid()}" + if sys.platform.startswith(("freebsd", "openbsd", "netbsd")): + path = f"/var/run/user/{getuid()}" + if not Path(path).exists(): + path = f"/tmp/runtime-{getuid()}" # noqa: S108 + else: + path = f"/run/user/{getuid()}" return self._append_app_name_and_version(path) @property @@ -168,13 +188,23 @@ def _first_item_as_path_if_multipath(self, directory: str) -> Path: return Path(directory) +def _get_user_media_dir(env_var: str, fallback_tilde_path: str) -> str: + media_dir = _get_user_dirs_folder(env_var) + if media_dir is None: + media_dir = os.environ.get(env_var, "").strip() + if not media_dir: + media_dir = os.path.expanduser(fallback_tilde_path) # noqa: PTH111 + + return media_dir + + def _get_user_dirs_folder(key: str) -> str | None: - """Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/""" - user_dirs_config_path = os.path.join(Unix().user_config_dir, "user-dirs.dirs") - if os.path.exists(user_dirs_config_path): + """Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/.""" + user_dirs_config_path = Path(Unix().user_config_dir) / "user-dirs.dirs" + if user_dirs_config_path.exists(): parser = ConfigParser() - with open(user_dirs_config_path) as stream: + with user_dirs_config_path.open() as stream: # Add fake section header, so ConfigParser doesn't complain parser.read_string(f"[top]\n{stream.read()}") @@ -183,8 +213,7 @@ def _get_user_dirs_folder(key: str) -> str | None: path = parser["top"][key].strip('"') # Handle relative home paths - path = path.replace("$HOME", os.path.expanduser("~")) - return path + return path.replace("$HOME", os.path.expanduser("~")) # noqa: PTH111 return None diff --git a/src/pip/_vendor/platformdirs/version.py b/src/pip/_vendor/platformdirs/version.py index d906a2c99e6..dc8c44cf7b2 100644 --- a/src/pip/_vendor/platformdirs/version.py +++ b/src/pip/_vendor/platformdirs/version.py @@ -1,4 +1,4 @@ # file generated by setuptools_scm # don't change, don't track in version control -__version__ = version = '3.2.0' -__version_tuple__ = version_tuple = (3, 2, 0) +__version__ = version = '3.8.1' +__version_tuple__ = version_tuple = (3, 8, 1) diff --git a/src/pip/_vendor/platformdirs/windows.py b/src/pip/_vendor/platformdirs/windows.py index e7573c3d6ae..b52c9c6ea89 100644 --- a/src/pip/_vendor/platformdirs/windows.py +++ b/src/pip/_vendor/platformdirs/windows.py @@ -1,16 +1,21 @@ +"""Windows.""" from __future__ import annotations import ctypes import os import sys from functools import lru_cache -from typing import Callable +from typing import TYPE_CHECKING from .api import PlatformDirsABC +if TYPE_CHECKING: + from collections.abc import Callable + class Windows(PlatformDirsABC): - """`MSDN on where to store app data files + """ + `MSDN on where to store app data files `_. Makes use of the `appname `, @@ -43,7 +48,7 @@ def _append_parts(self, path: str, *, opinion_value: str | None = None) -> str: params.append(opinion_value) if self.version: params.append(self.version) - path = os.path.join(path, *params) + path = os.path.join(path, *params) # noqa: PTH118 self._optionally_create_directory(path) return path @@ -85,36 +90,53 @@ def user_state_dir(self) -> str: @property def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it - """ + """:return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it""" path = self.user_data_dir if self.opinion: - path = os.path.join(path, "Logs") + path = os.path.join(path, "Logs") # noqa: PTH118 self._optionally_create_directory(path) return path @property def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents`` - """ + """:return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents``""" return os.path.normpath(get_win_folder("CSIDL_PERSONAL")) + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user e.g. ``%USERPROFILE%\\Downloads``""" + return os.path.normpath(get_win_folder("CSIDL_DOWNLOADS")) + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user e.g. ``%USERPROFILE%\\Pictures``""" + return os.path.normpath(get_win_folder("CSIDL_MYPICTURES")) + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user e.g. ``%USERPROFILE%\\Videos``""" + return os.path.normpath(get_win_folder("CSIDL_MYVIDEO")) + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user e.g. ``%USERPROFILE%\\Music``""" + return os.path.normpath(get_win_folder("CSIDL_MYMUSIC")) + @property def user_runtime_dir(self) -> str: """ :return: runtime directory tied to the user, e.g. ``%USERPROFILE%\\AppData\\Local\\Temp\\$appauthor\\$appname`` """ - path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp")) + path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp")) # noqa: PTH118 return self._append_parts(path) def get_win_folder_from_env_vars(csidl_name: str) -> str: """Get folder from environment variables.""" - if csidl_name == "CSIDL_PERSONAL": # does not have an environment name - return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents") + result = get_win_folder_if_csidl_name_not_env_var(csidl_name) + if result is not None: + return result env_var_name = { "CSIDL_APPDATA": "APPDATA", @@ -122,28 +144,54 @@ def get_win_folder_from_env_vars(csidl_name: str) -> str: "CSIDL_LOCAL_APPDATA": "LOCALAPPDATA", }.get(csidl_name) if env_var_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) result = os.environ.get(env_var_name) if result is None: - raise ValueError(f"Unset environment variable: {env_var_name}") + msg = f"Unset environment variable: {env_var_name}" + raise ValueError(msg) return result +def get_win_folder_if_csidl_name_not_env_var(csidl_name: str) -> str | None: + """Get folder for a CSIDL name that does not exist as an environment variable.""" + if csidl_name == "CSIDL_PERSONAL": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents") # noqa: PTH118 + + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Downloads") # noqa: PTH118 + + if csidl_name == "CSIDL_MYPICTURES": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Pictures") # noqa: PTH118 + + if csidl_name == "CSIDL_MYVIDEO": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Videos") # noqa: PTH118 + + if csidl_name == "CSIDL_MYMUSIC": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Music") # noqa: PTH118 + return None + + def get_win_folder_from_registry(csidl_name: str) -> str: - """Get folder from the registry. + """ + Get folder from the registry. - This is a fallback technique at best. I'm not sure if using the - registry for this guarantees us the correct answer for all CSIDL_* - names. + This is a fallback technique at best. I'm not sure if using the registry for these guarantees us the correct answer + for all CSIDL_* names. """ shell_folder_name = { "CSIDL_APPDATA": "AppData", "CSIDL_COMMON_APPDATA": "Common AppData", "CSIDL_LOCAL_APPDATA": "Local AppData", "CSIDL_PERSONAL": "Personal", + "CSIDL_DOWNLOADS": "{374DE290-123F-4565-9164-39C4925E467B}", + "CSIDL_MYPICTURES": "My Pictures", + "CSIDL_MYVIDEO": "My Video", + "CSIDL_MYMUSIC": "My Music", }.get(csidl_name) if shell_folder_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) if sys.platform != "win32": # only needed for mypy type checker to know that this code runs only on Windows raise NotImplementedError import winreg @@ -155,25 +203,37 @@ def get_win_folder_from_registry(csidl_name: str) -> str: def get_win_folder_via_ctypes(csidl_name: str) -> str: """Get folder with ctypes.""" + # There is no 'CSIDL_DOWNLOADS'. + # Use 'CSIDL_PROFILE' (40) and append the default folder 'Downloads' instead. + # https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid + csidl_const = { "CSIDL_APPDATA": 26, "CSIDL_COMMON_APPDATA": 35, "CSIDL_LOCAL_APPDATA": 28, "CSIDL_PERSONAL": 5, + "CSIDL_MYPICTURES": 39, + "CSIDL_MYVIDEO": 14, + "CSIDL_MYMUSIC": 13, + "CSIDL_DOWNLOADS": 40, }.get(csidl_name) if csidl_const is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) buf = ctypes.create_unicode_buffer(1024) windll = getattr(ctypes, "windll") # noqa: B009 # using getattr to avoid false positive with mypy type checker windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) # Downgrade to short path name if it has highbit chars. - if any(ord(c) > 255 for c in buf): + if any(ord(c) > 255 for c in buf): # noqa: PLR2004 buf2 = ctypes.create_unicode_buffer(1024) if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): buf = buf2 + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(buf.value, "Downloads") # noqa: PTH118 + return buf.value diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index dcf89dc04c5..07671fb58af 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -4,7 +4,7 @@ distlib==0.3.6 distro==1.8.0 msgpack==1.0.5 packaging==21.3 -platformdirs==3.2.0 +platformdirs==3.8.1 pyparsing==3.0.9 pyproject-hooks==1.0.0 requests==2.31.0 From 6ee8884ac4a241f665aa712bcbe0e185012bccdb Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 9 Jul 2023 21:30:39 +0100 Subject: [PATCH 64/70] Upgrade pyparsing to 3.1.0 --- news/pyparsing.vendor.rst | 1 + src/pip/_vendor/pyparsing/__init__.py | 75 +- src/pip/_vendor/pyparsing/actions.py | 34 +- src/pip/_vendor/pyparsing/common.py | 58 +- src/pip/_vendor/pyparsing/core.py | 1197 +++++++++++------ src/pip/_vendor/pyparsing/diagram/__init__.py | 32 +- src/pip/_vendor/pyparsing/exceptions.py | 64 +- src/pip/_vendor/pyparsing/helpers.py | 196 +-- src/pip/_vendor/pyparsing/results.py | 128 +- src/pip/_vendor/pyparsing/testing.py | 24 +- src/pip/_vendor/pyparsing/unicode.py | 101 +- src/pip/_vendor/pyparsing/util.py | 89 +- src/pip/_vendor/vendor.txt | 2 +- 13 files changed, 1232 insertions(+), 769 deletions(-) create mode 100644 news/pyparsing.vendor.rst diff --git a/news/pyparsing.vendor.rst b/news/pyparsing.vendor.rst new file mode 100644 index 00000000000..9feffb2460f --- /dev/null +++ b/news/pyparsing.vendor.rst @@ -0,0 +1 @@ +Upgrade pyparsing to 3.1.0 diff --git a/src/pip/_vendor/pyparsing/__init__.py b/src/pip/_vendor/pyparsing/__init__.py index 75372500ed9..88bc10ac18a 100644 --- a/src/pip/_vendor/pyparsing/__init__.py +++ b/src/pip/_vendor/pyparsing/__init__.py @@ -56,7 +56,7 @@ :class:`'|'`, :class:`'^'` and :class:`'&'` operators. The :class:`ParseResults` object returned from -:class:`ParserElement.parseString` can be +:class:`ParserElement.parse_string` can be accessed as a nested list, a dictionary, or an object with named attributes. @@ -85,11 +85,11 @@ and :class:`'&'` operators to combine simple expressions into more complex ones - associate names with your parsed results using - :class:`ParserElement.setResultsName` + :class:`ParserElement.set_results_name` - access the parsed data, which is returned as a :class:`ParseResults` object - - find some helpful expression short-cuts like :class:`delimitedList` - and :class:`oneOf` + - find some helpful expression short-cuts like :class:`DelimitedList` + and :class:`one_of` - find more useful common expressions in the :class:`pyparsing_common` namespace class """ @@ -106,30 +106,22 @@ class version_info(NamedTuple): @property def __version__(self): return ( - "{}.{}.{}".format(self.major, self.minor, self.micro) + f"{self.major}.{self.minor}.{self.micro}" + ( - "{}{}{}".format( - "r" if self.releaselevel[0] == "c" else "", - self.releaselevel[0], - self.serial, - ), + f"{'r' if self.releaselevel[0] == 'c' else ''}{self.releaselevel[0]}{self.serial}", "", )[self.releaselevel == "final"] ) def __str__(self): - return "{} {} / {}".format(__name__, self.__version__, __version_time__) + return f"{__name__} {self.__version__} / {__version_time__}" def __repr__(self): - return "{}.{}({})".format( - __name__, - type(self).__name__, - ", ".join("{}={!r}".format(*nv) for nv in zip(self._fields, self)), - ) + return f"{__name__}.{type(self).__name__}({', '.join('{}={!r}'.format(*nv) for nv in zip(self._fields, self))})" -__version_info__ = version_info(3, 0, 9, "final", 0) -__version_time__ = "05 May 2022 07:02 UTC" +__version_info__ = version_info(3, 1, 0, "final", 1) +__version_time__ = "18 Jun 2023 14:05 UTC" __version__ = __version_info__.__version__ __versionTime__ = __version_time__ __author__ = "Paul McGuire " @@ -139,9 +131,9 @@ def __repr__(self): from .actions import * from .core import __diag__, __compat__ from .results import * -from .core import * +from .core import * # type: ignore[misc, assignment] from .core import _builtin_exprs as core_builtin_exprs -from .helpers import * +from .helpers import * # type: ignore[misc, assignment] from .helpers import _builtin_exprs as helper_builtin_exprs from .unicode import unicode_set, UnicodeRangeList, pyparsing_unicode as unicode @@ -153,11 +145,11 @@ def __repr__(self): # define backward compat synonyms if "pyparsing_unicode" not in globals(): - pyparsing_unicode = unicode + pyparsing_unicode = unicode # type: ignore[misc] if "pyparsing_common" not in globals(): - pyparsing_common = common + pyparsing_common = common # type: ignore[misc] if "pyparsing_test" not in globals(): - pyparsing_test = testing + pyparsing_test = testing # type: ignore[misc] core_builtin_exprs += common_builtin_exprs + helper_builtin_exprs @@ -174,7 +166,9 @@ def __repr__(self): "CaselessKeyword", "CaselessLiteral", "CharsNotIn", + "CloseMatch", "Combine", + "DelimitedList", "Dict", "Each", "Empty", @@ -227,9 +221,11 @@ def __repr__(self): "alphas8bit", "any_close_tag", "any_open_tag", + "autoname_elements", "c_style_comment", "col", "common_html_entity", + "condition_as_parse_action", "counted_array", "cpp_style_comment", "dbl_quoted_string", @@ -241,6 +237,7 @@ def __repr__(self): "html_comment", "identchars", "identbodychars", + "infix_notation", "java_style_comment", "line", "line_end", @@ -255,8 +252,12 @@ def __repr__(self): "null_debug_action", "nums", "one_of", + "original_text_for", "printables", "punc8bit", + "pyparsing_common", + "pyparsing_test", + "pyparsing_unicode", "python_style_comment", "quoted_string", "remove_quotes", @@ -267,28 +268,20 @@ def __repr__(self): "srange", "string_end", "string_start", + "token_map", "trace_parse_action", + "ungroup", + "unicode_set", "unicode_string", "with_attribute", - "indentedBlock", - "original_text_for", - "ungroup", - "infix_notation", - "locatedExpr", "with_class", - "CloseMatch", - "token_map", - "pyparsing_common", - "pyparsing_unicode", - "unicode_set", - "condition_as_parse_action", - "pyparsing_test", # pre-PEP8 compatibility names "__versionTime__", "anyCloseTag", "anyOpenTag", "cStyleComment", "commonHTMLEntity", + "conditionAsParseAction", "countedArray", "cppStyleComment", "dblQuotedString", @@ -296,9 +289,12 @@ def __repr__(self): "delimitedList", "dictOf", "htmlComment", + "indentedBlock", + "infixNotation", "javaStyleComment", "lineEnd", "lineStart", + "locatedExpr", "makeHTMLTags", "makeXMLTags", "matchOnlyAtCol", @@ -308,6 +304,7 @@ def __repr__(self): "nullDebugAction", "oneOf", "opAssoc", + "originalTextFor", "pythonStyleComment", "quotedString", "removeQuotes", @@ -317,15 +314,9 @@ def __repr__(self): "sglQuotedString", "stringEnd", "stringStart", + "tokenMap", "traceParseAction", "unicodeString", "withAttribute", - "indentedBlock", - "originalTextFor", - "infixNotation", - "locatedExpr", "withClass", - "tokenMap", - "conditionAsParseAction", - "autoname_elements", ] diff --git a/src/pip/_vendor/pyparsing/actions.py b/src/pip/_vendor/pyparsing/actions.py index f72c66e7431..ca6e4c6afb4 100644 --- a/src/pip/_vendor/pyparsing/actions.py +++ b/src/pip/_vendor/pyparsing/actions.py @@ -1,7 +1,7 @@ # actions.py from .exceptions import ParseException -from .util import col +from .util import col, replaced_by_pep8 class OnlyOnce: @@ -38,7 +38,7 @@ def match_only_at_col(n): def verify_col(strg, locn, toks): if col(locn, strg) != n: - raise ParseException(strg, locn, "matched token not at column {}".format(n)) + raise ParseException(strg, locn, f"matched token not at column {n}") return verify_col @@ -148,15 +148,13 @@ def pa(s, l, tokens): raise ParseException( s, l, - "attribute {!r} has value {!r}, must be {!r}".format( - attrName, tokens[attrName], attrValue - ), + f"attribute {attrName!r} has value {tokens[attrName]!r}, must be {attrValue!r}", ) return pa -with_attribute.ANY_VALUE = object() +with_attribute.ANY_VALUE = object() # type: ignore [attr-defined] def with_class(classname, namespace=""): @@ -195,13 +193,25 @@ def with_class(classname, namespace=""): 1 4 0 1 0 1,3 2,3 1,1 """ - classattr = "{}:class".format(namespace) if namespace else "class" + classattr = f"{namespace}:class" if namespace else "class" return with_attribute(**{classattr: classname}) # pre-PEP8 compatibility symbols -replaceWith = replace_with -removeQuotes = remove_quotes -withAttribute = with_attribute -withClass = with_class -matchOnlyAtCol = match_only_at_col +# fmt: off +@replaced_by_pep8(replace_with) +def replaceWith(): ... + +@replaced_by_pep8(remove_quotes) +def removeQuotes(): ... + +@replaced_by_pep8(with_attribute) +def withAttribute(): ... + +@replaced_by_pep8(with_class) +def withClass(): ... + +@replaced_by_pep8(match_only_at_col) +def matchOnlyAtCol(): ... + +# fmt: on diff --git a/src/pip/_vendor/pyparsing/common.py b/src/pip/_vendor/pyparsing/common.py index 1859fb79cc4..7a666b276df 100644 --- a/src/pip/_vendor/pyparsing/common.py +++ b/src/pip/_vendor/pyparsing/common.py @@ -1,6 +1,6 @@ # common.py from .core import * -from .helpers import delimited_list, any_open_tag, any_close_tag +from .helpers import DelimitedList, any_open_tag, any_close_tag from datetime import datetime @@ -22,17 +22,17 @@ class pyparsing_common: Parse actions: - - :class:`convertToInteger` - - :class:`convertToFloat` - - :class:`convertToDate` - - :class:`convertToDatetime` - - :class:`stripHTMLTags` - - :class:`upcaseTokens` - - :class:`downcaseTokens` + - :class:`convert_to_integer` + - :class:`convert_to_float` + - :class:`convert_to_date` + - :class:`convert_to_datetime` + - :class:`strip_html_tags` + - :class:`upcase_tokens` + - :class:`downcase_tokens` Example:: - pyparsing_common.number.runTests(''' + pyparsing_common.number.run_tests(''' # any int or real number, returned as the appropriate type 100 -100 @@ -42,7 +42,7 @@ class pyparsing_common: 1e-12 ''') - pyparsing_common.fnumber.runTests(''' + pyparsing_common.fnumber.run_tests(''' # any int or real number, returned as float 100 -100 @@ -52,19 +52,19 @@ class pyparsing_common: 1e-12 ''') - pyparsing_common.hex_integer.runTests(''' + pyparsing_common.hex_integer.run_tests(''' # hex numbers 100 FF ''') - pyparsing_common.fraction.runTests(''' + pyparsing_common.fraction.run_tests(''' # fractions 1/2 -3/4 ''') - pyparsing_common.mixed_integer.runTests(''' + pyparsing_common.mixed_integer.run_tests(''' # mixed fractions 1 1/2 @@ -73,8 +73,8 @@ class pyparsing_common: ''') import uuid - pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) - pyparsing_common.uuid.runTests(''' + pyparsing_common.uuid.set_parse_action(token_map(uuid.UUID)) + pyparsing_common.uuid.run_tests(''' # uuid 12345678-1234-5678-1234-567812345678 ''') @@ -260,8 +260,8 @@ def convert_to_date(fmt: str = "%Y-%m-%d"): Example:: date_expr = pyparsing_common.iso8601_date.copy() - date_expr.setParseAction(pyparsing_common.convertToDate()) - print(date_expr.parseString("1999-12-31")) + date_expr.set_parse_action(pyparsing_common.convert_to_date()) + print(date_expr.parse_string("1999-12-31")) prints:: @@ -287,8 +287,8 @@ def convert_to_datetime(fmt: str = "%Y-%m-%dT%H:%M:%S.%f"): Example:: dt_expr = pyparsing_common.iso8601_datetime.copy() - dt_expr.setParseAction(pyparsing_common.convertToDatetime()) - print(dt_expr.parseString("1999-12-31T23:59:59.999")) + dt_expr.set_parse_action(pyparsing_common.convert_to_datetime()) + print(dt_expr.parse_string("1999-12-31T23:59:59.999")) prints:: @@ -326,9 +326,9 @@ def strip_html_tags(s: str, l: int, tokens: ParseResults): # strip HTML links from normal text text = 'More info at the pyparsing wiki page' - td, td_end = makeHTMLTags("TD") - table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - print(table_text.parseString(text).body) + td, td_end = make_html_tags("TD") + table_text = td + SkipTo(td_end).set_parse_action(pyparsing_common.strip_html_tags)("body") + td_end + print(table_text.parse_string(text).body) Prints:: @@ -348,7 +348,7 @@ def strip_html_tags(s: str, l: int, tokens: ParseResults): .streamline() .set_name("commaItem") ) - comma_separated_list = delimited_list( + comma_separated_list = DelimitedList( Opt(quoted_string.copy() | _commasepitem, default="") ).set_name("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" @@ -363,7 +363,7 @@ def strip_html_tags(s: str, l: int, tokens: ParseResults): url = Regex( # https://mathiasbynens.be/demo/url-regex # https://gist.github.com/dperini/729294 - r"^" + + r"(?P" + # protocol identifier (optional) # short syntax // still required r"(?:(?:(?Phttps?|ftp):)?\/\/)" + @@ -405,18 +405,26 @@ def strip_html_tags(s: str, l: int, tokens: ParseResults): r"(\?(?P[^#]*))?" + # fragment (optional) r"(#(?P\S*))?" + - r"$" + r")" ).set_name("url") + """URL (http/https/ftp scheme)""" # fmt: on # pre-PEP8 compatibility names convertToInteger = convert_to_integer + """Deprecated - use :class:`convert_to_integer`""" convertToFloat = convert_to_float + """Deprecated - use :class:`convert_to_float`""" convertToDate = convert_to_date + """Deprecated - use :class:`convert_to_date`""" convertToDatetime = convert_to_datetime + """Deprecated - use :class:`convert_to_datetime`""" stripHTMLTags = strip_html_tags + """Deprecated - use :class:`strip_html_tags`""" upcaseTokens = upcase_tokens + """Deprecated - use :class:`upcase_tokens`""" downcaseTokens = downcase_tokens + """Deprecated - use :class:`downcase_tokens`""" _builtin_exprs = [ diff --git a/src/pip/_vendor/pyparsing/core.py b/src/pip/_vendor/pyparsing/core.py index 6ff3c766f7d..8d5a856ecd6 100644 --- a/src/pip/_vendor/pyparsing/core.py +++ b/src/pip/_vendor/pyparsing/core.py @@ -1,19 +1,22 @@ # # core.py # + +from collections import deque import os import typing from typing import ( - NamedTuple, - Union, - Callable, Any, + Callable, Generator, - Tuple, List, - TextIO, - Set, + NamedTuple, Sequence, + Set, + TextIO, + Tuple, + Union, + cast, ) from abc import ABC, abstractmethod from enum import Enum @@ -40,6 +43,7 @@ _flatten, LRUMemo as _LRUMemo, UnboundedMemo as _UnboundedMemo, + replaced_by_pep8, ) from .exceptions import * from .actions import * @@ -134,6 +138,7 @@ def enable_all_warnings(cls) -> None: class Diagnostics(Enum): """ Diagnostic configuration (all default to disabled) + - ``warn_multiple_tokens_in_named_alternation`` - flag to enable warnings when a results name is defined on a :class:`MatchFirst` or :class:`Or` expression with one or more :class:`And` subexpressions - ``warn_ungrouped_named_tokens_in_collection`` - flag to enable warnings when a results @@ -228,6 +233,8 @@ def _should_enable_warnings( } _generatorType = types.GeneratorType +ParseImplReturnType = Tuple[int, Any] +PostParseReturnType = Union[ParseResults, Sequence[ParseResults]] ParseAction = Union[ Callable[[], Any], Callable[[ParseResults], Any], @@ -256,7 +263,7 @@ def _should_enable_warnings( alphanums = alphas + nums printables = "".join([c for c in string.printable if c not in string.whitespace]) -_trim_arity_call_line: traceback.StackSummary = None +_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment] def _trim_arity(func, max_limit=3): @@ -269,11 +276,6 @@ def _trim_arity(func, max_limit=3): limit = 0 found_arity = False - def extract_tb(tb, limit=0): - frames = traceback.extract_tb(tb, limit=limit) - frame_summary = frames[-1] - return [frame_summary[:2]] - # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time @@ -297,8 +299,10 @@ def wrapper(*args): raise else: tb = te.__traceback__ + frames = traceback.extract_tb(tb, limit=2) + frame_summary = frames[-1] trim_arity_type_error = ( - extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth + [frame_summary[:2]][-1][:2] == pa_call_line_synth ) del tb @@ -320,7 +324,7 @@ def wrapper(*args): def condition_as_parse_action( - fn: ParseCondition, message: str = None, fatal: bool = False + fn: ParseCondition, message: typing.Optional[str] = None, fatal: bool = False ) -> ParseAction: """ Function to convert a simple predicate function that returns ``True`` or ``False`` @@ -353,15 +357,9 @@ def _default_start_debug_action( cache_hit_str = "*" if cache_hit else "" print( ( - "{}Match {} at loc {}({},{})\n {}\n {}^".format( - cache_hit_str, - expr, - loc, - lineno(loc, instring), - col(loc, instring), - line(loc, instring), - " " * (col(loc, instring) - 1), - ) + f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{col(loc, instring)})\n" + f" {line(loc, instring)}\n" + f" {' ' * (col(loc, instring) - 1)}^" ) ) @@ -375,7 +373,7 @@ def _default_success_debug_action( cache_hit: bool = False, ): cache_hit_str = "*" if cache_hit else "" - print("{}Matched {} -> {}".format(cache_hit_str, expr, toks.as_list())) + print(f"{cache_hit_str}Matched {expr} -> {toks.as_list()}") def _default_exception_debug_action( @@ -386,11 +384,7 @@ def _default_exception_debug_action( cache_hit: bool = False, ): cache_hit_str = "*" if cache_hit else "" - print( - "{}Match {} failed, {} raised: {}".format( - cache_hit_str, expr, type(exc).__name__, exc - ) - ) + print(f"{cache_hit_str}Match {expr} failed, {type(exc).__name__} raised: {exc}") def null_debug_action(*args): @@ -402,7 +396,7 @@ class ParserElement(ABC): DEFAULT_WHITE_CHARS: str = " \n\t\r" verbose_stacktrace: bool = False - _literalStringClass: typing.Optional[type] = None + _literalStringClass: type = None # type: ignore[assignment] @staticmethod def set_default_whitespace_chars(chars: str) -> None: @@ -447,6 +441,18 @@ def inline_literals_using(cls: type) -> None: """ ParserElement._literalStringClass = cls + @classmethod + def using_each(cls, seq, **class_kwargs): + """ + Yields a sequence of class(obj, **class_kwargs) for obj in seq. + + Example:: + + LPAR, RPAR, LBRACE, RBRACE, SEMI = Suppress.using_each("(){};") + + """ + yield from (cls(obj, **class_kwargs) for obj in seq) + class DebugActions(NamedTuple): debug_try: typing.Optional[DebugStartAction] debug_match: typing.Optional[DebugSuccessAction] @@ -455,9 +461,9 @@ class DebugActions(NamedTuple): def __init__(self, savelist: bool = False): self.parseAction: List[ParseAction] = list() self.failAction: typing.Optional[ParseFailAction] = None - self.customName = None - self._defaultName = None - self.resultsName = None + self.customName: str = None # type: ignore[assignment] + self._defaultName: typing.Optional[str] = None + self.resultsName: str = None # type: ignore[assignment] self.saveAsList = savelist self.skipWhitespace = True self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) @@ -490,12 +496,29 @@ def suppress_warning(self, warning_type: Diagnostics) -> "ParserElement": base.suppress_warning(Diagnostics.warn_on_parse_using_empty_Forward) # statement would normally raise a warning, but is now suppressed - print(base.parseString("x")) + print(base.parse_string("x")) """ self.suppress_warnings_.append(warning_type) return self + def visit_all(self): + """General-purpose method to yield all expressions and sub-expressions + in a grammar. Typically just for internal use. + """ + to_visit = deque([self]) + seen = set() + while to_visit: + cur = to_visit.popleft() + + # guard against looping forever through recursive grammars + if cur in seen: + continue + seen.add(cur) + + to_visit.extend(cur.recurse()) + yield cur + def copy(self) -> "ParserElement": """ Make a copy of this :class:`ParserElement`. Useful for defining @@ -585,11 +608,11 @@ def breaker(instring, loc, doActions=True, callPreParse=True): pdb.set_trace() return _parseMethod(instring, loc, doActions, callPreParse) - breaker._originalParseMethod = _parseMethod - self._parse = breaker + breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined] + self._parse = breaker # type: ignore [assignment] else: if hasattr(self._parse, "_originalParseMethod"): - self._parse = self._parse._originalParseMethod + self._parse = self._parse._originalParseMethod # type: ignore [attr-defined, assignment] return self def set_parse_action(self, *fns: ParseAction, **kwargs) -> "ParserElement": @@ -601,9 +624,9 @@ def set_parse_action(self, *fns: ParseAction, **kwargs) -> "ParserElement": Each parse action ``fn`` is a callable method with 0-3 arguments, called as ``fn(s, loc, toks)`` , ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + - ``s`` = the original string being parsed (see note below) + - ``loc`` = the location of the matching substring + - ``toks`` = a list of the matched tokens, packaged as a :class:`ParseResults` object The parsed tokens are passed to the parse action as ParseResults. They can be modified in place using list-style append, extend, and pop operations to update @@ -621,7 +644,7 @@ def set_parse_action(self, *fns: ParseAction, **kwargs) -> "ParserElement": Optional keyword arguments: - - call_during_try = (default= ``False``) indicate if parse action should be run during + - ``call_during_try`` = (default= ``False``) indicate if parse action should be run during lookaheads and alternate testing. For parse actions that have side effects, it is important to only call the parse action once it is determined that it is being called as part of a successful parse. For parse actions that perform additional @@ -697,10 +720,10 @@ def add_condition(self, *fns: ParseCondition, **kwargs) -> "ParserElement": Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise + - ``message`` = define a custom message to be used in the raised exception + - ``fatal`` = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - - call_during_try = boolean to indicate if this method should be called during internal tryParse calls, + - ``call_during_try`` = boolean to indicate if this method should be called during internal tryParse calls, default=False Example:: @@ -716,7 +739,9 @@ def add_condition(self, *fns: ParseCondition, **kwargs) -> "ParserElement": for fn in fns: self.parseAction.append( condition_as_parse_action( - fn, message=kwargs.get("message"), fatal=kwargs.get("fatal", False) + fn, + message=str(kwargs.get("message")), + fatal=bool(kwargs.get("fatal", False)), ) ) @@ -731,30 +756,33 @@ def set_fail_action(self, fn: ParseFailAction) -> "ParserElement": Fail acton fn is a callable function that takes the arguments ``fn(s, loc, expr, err)`` where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown + - ``s`` = string being parsed + - ``loc`` = location where expression match was attempted and failed + - ``expr`` = the parse expression that failed + - ``err`` = the exception thrown The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self - def _skipIgnorables(self, instring, loc): + def _skipIgnorables(self, instring: str, loc: int) -> int: + if not self.ignoreExprs: + return loc exprsFound = True + ignore_expr_fns = [e._parse for e in self.ignoreExprs] while exprsFound: exprsFound = False - for e in self.ignoreExprs: + for ignore_fn in ignore_expr_fns: try: while 1: - loc, dummy = e._parse(instring, loc) + loc, dummy = ignore_fn(instring, loc) exprsFound = True except ParseException: pass return loc - def preParse(self, instring, loc): + def preParse(self, instring: str, loc: int) -> int: if self.ignoreExprs: loc = self._skipIgnorables(instring, loc) @@ -830,7 +858,7 @@ def _parseNoCache( try: for fn in self.parseAction: try: - tokens = fn(instring, tokens_start, ret_tokens) + tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type] except IndexError as parse_action_exc: exc = ParseException("exception raised in parse action") raise exc from parse_action_exc @@ -853,7 +881,7 @@ def _parseNoCache( else: for fn in self.parseAction: try: - tokens = fn(instring, tokens_start, ret_tokens) + tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type] except IndexError as parse_action_exc: exc = ParseException("exception raised in parse action") raise exc from parse_action_exc @@ -875,17 +903,24 @@ def _parseNoCache( return loc, ret_tokens - def try_parse(self, instring: str, loc: int, raise_fatal: bool = False) -> int: + def try_parse( + self, + instring: str, + loc: int, + *, + raise_fatal: bool = False, + do_actions: bool = False, + ) -> int: try: - return self._parse(instring, loc, doActions=False)[0] + return self._parse(instring, loc, doActions=do_actions)[0] except ParseFatalException: if raise_fatal: raise raise ParseException(instring, loc, self.errmsg, self) - def can_parse_next(self, instring: str, loc: int) -> bool: + def can_parse_next(self, instring: str, loc: int, do_actions: bool = False) -> bool: try: - self.try_parse(instring, loc) + self.try_parse(instring, loc, do_actions=do_actions) except (ParseException, IndexError): return False else: @@ -897,10 +932,23 @@ def can_parse_next(self, instring: str, loc: int) -> bool: Tuple[int, "Forward", bool], Tuple[int, Union[ParseResults, Exception]] ] = {} + class _CacheType(dict): + """ + class to help type checking + """ + + not_in_cache: bool + + def get(self, *args): + ... + + def set(self, *args): + ... + # argument cache for optimizing repeated calls when backtracking through recursive expressions packrat_cache = ( - {} - ) # this is set later by enabled_packrat(); this is here so that reset_cache() doesn't fail + _CacheType() + ) # set later by enable_packrat(); this is here so that reset_cache() doesn't fail packrat_cache_lock = RLock() packrat_cache_stats = [0, 0] @@ -930,24 +978,25 @@ def _parseCache( ParserElement.packrat_cache_stats[HIT] += 1 if self.debug and self.debugActions.debug_try: try: - self.debugActions.debug_try(instring, loc, self, cache_hit=True) + self.debugActions.debug_try(instring, loc, self, cache_hit=True) # type: ignore [call-arg] except TypeError: pass if isinstance(value, Exception): if self.debug and self.debugActions.debug_fail: try: self.debugActions.debug_fail( - instring, loc, self, value, cache_hit=True + instring, loc, self, value, cache_hit=True # type: ignore [call-arg] ) except TypeError: pass raise value + value = cast(Tuple[int, ParseResults, int], value) loc_, result, endloc = value[0], value[1].copy(), value[2] if self.debug and self.debugActions.debug_match: try: self.debugActions.debug_match( - instring, loc_, endloc, self, result, cache_hit=True + instring, loc_, endloc, self, result, cache_hit=True # type: ignore [call-arg] ) except TypeError: pass @@ -1009,7 +1058,7 @@ def enable_left_recursion( Parameters: - - cache_size_limit - (default=``None``) - memoize at most this many + - ``cache_size_limit`` - (default=``None``) - memoize at most this many ``Forward`` elements during matching; if ``None`` (the default), memoize all ``Forward`` elements. @@ -1022,9 +1071,9 @@ def enable_left_recursion( elif ParserElement._packratEnabled: raise RuntimeError("Packrat and Bounded Recursion are not compatible") if cache_size_limit is None: - ParserElement.recursion_memos = _UnboundedMemo() + ParserElement.recursion_memos = _UnboundedMemo() # type: ignore[assignment] elif cache_size_limit > 0: - ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) + ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment] else: raise NotImplementedError("Memo size of %s" % cache_size_limit) ParserElement._left_recursion_enabled = True @@ -1040,7 +1089,7 @@ def enable_packrat(cache_size_limit: int = 128, *, force: bool = False) -> None: Parameters: - - cache_size_limit - (default= ``128``) - if an integer value is provided + - ``cache_size_limit`` - (default= ``128``) - if an integer value is provided will limit the size of the packrat cache; if None is passed, then the cache size will be unbounded; if 0 is passed, the cache will be effectively disabled. @@ -1070,7 +1119,7 @@ def enable_packrat(cache_size_limit: int = 128, *, force: bool = False) -> None: if cache_size_limit is None: ParserElement.packrat_cache = _UnboundedCache() else: - ParserElement.packrat_cache = _FifoCache(cache_size_limit) + ParserElement.packrat_cache = _FifoCache(cache_size_limit) # type: ignore[assignment] ParserElement._parse = ParserElement._parseCache def parse_string( @@ -1088,7 +1137,7 @@ def parse_string( an object with attributes if the given parser includes results names. If the input string is required to match the entire grammar, ``parse_all`` flag must be set to ``True``. This - is also equivalent to ending the grammar with :class:`StringEnd`(). + is also equivalent to ending the grammar with :class:`StringEnd`\\ (). To report proper column numbers, ``parse_string`` operates on a copy of the input string where all tabs are converted to spaces (8 spaces per tab, as per the default in ``string.expandtabs``). If the input string @@ -1198,7 +1247,9 @@ def scan_string( try: while loc <= instrlen and matches < maxMatches: try: - preloc = preparseFn(instring, loc) + preloc: int = preparseFn(instring, loc) + nextLoc: int + tokens: ParseResults nextLoc, tokens = parseFn(instring, preloc, callPreParse=False) except ParseException: loc = preloc + 1 @@ -1352,7 +1403,7 @@ def split( def __add__(self, other) -> "ParserElement": """ Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement` - converts them to :class:`Literal`s by default. + converts them to :class:`Literal`\\ s by default. Example:: @@ -1364,11 +1415,11 @@ def __add__(self, other) -> "ParserElement": Hello, World! -> ['Hello', ',', 'World', '!'] - ``...`` may be used as a parse expression as a short form of :class:`SkipTo`. + ``...`` may be used as a parse expression as a short form of :class:`SkipTo`:: Literal('start') + ... + Literal('end') - is equivalent to: + is equivalent to:: Literal('start') + SkipTo('end')("_skipped*") + Literal('end') @@ -1382,11 +1433,7 @@ def __add__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return And([self, other]) def __radd__(self, other) -> "ParserElement": @@ -1399,11 +1446,7 @@ def __radd__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other + self def __sub__(self, other) -> "ParserElement": @@ -1413,11 +1456,7 @@ def __sub__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return self + And._ErrorStop() + other def __rsub__(self, other) -> "ParserElement": @@ -1427,11 +1466,7 @@ def __rsub__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other - self def __mul__(self, other) -> "ParserElement": @@ -1440,11 +1475,12 @@ def __mul__(self, other) -> "ParserElement": ``expr + expr + expr``. Expressions may also be multiplied by a 2-integer tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples may also include ``None`` as in: + - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent - to ``expr*n + ZeroOrMore(expr)`` - (read as "at least n instances of ``expr``") + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") - ``expr*(None, n)`` is equivalent to ``expr*(0, n)`` - (read as "0 to n instances of ``expr``") + (read as "0 to n instances of ``expr``") - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)`` - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)`` @@ -1477,17 +1513,9 @@ def __mul__(self, other) -> "ParserElement": minElements, optElements = other optElements -= minElements else: - raise TypeError( - "cannot multiply ParserElement and ({}) objects".format( - ",".join(type(item).__name__ for item in other) - ) - ) + return NotImplemented else: - raise TypeError( - "cannot multiply ParserElement and {} objects".format( - type(other).__name__ - ) - ) + return NotImplemented if minElements < 0: raise ValueError("cannot multiply ParserElement by negative value") @@ -1531,13 +1559,12 @@ def __or__(self, other) -> "ParserElement": return _PendingSkip(self, must_skip=True) if isinstance(other, str_type): + # `expr | ""` is equivalent to `Opt(expr)` + if other == "": + return Opt(self) other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return MatchFirst([self, other]) def __ror__(self, other) -> "ParserElement": @@ -1547,11 +1574,7 @@ def __ror__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other | self def __xor__(self, other) -> "ParserElement": @@ -1561,11 +1584,7 @@ def __xor__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return Or([self, other]) def __rxor__(self, other) -> "ParserElement": @@ -1575,11 +1594,7 @@ def __rxor__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other ^ self def __and__(self, other) -> "ParserElement": @@ -1589,11 +1604,7 @@ def __and__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return Each([self, other]) def __rand__(self, other) -> "ParserElement": @@ -1603,11 +1614,7 @@ def __rand__(self, other) -> "ParserElement": if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other & self def __invert__(self) -> "ParserElement": @@ -1636,38 +1643,58 @@ def __getitem__(self, key): ``None`` may be used in place of ``...``. - Note that ``expr[..., n]`` and ``expr[m, n]``do not raise an exception - if more than ``n`` ``expr``s exist in the input stream. If this behavior is + Note that ``expr[..., n]`` and ``expr[m, n]`` do not raise an exception + if more than ``n`` ``expr``\\ s exist in the input stream. If this behavior is desired, then write ``expr[..., n] + ~expr``. + + For repetition with a stop_on expression, use slice notation: + + - ``expr[...: end_expr]`` and ``expr[0, ...: end_expr]`` are equivalent to ``ZeroOrMore(expr, stop_on=end_expr)`` + - ``expr[1, ...: end_expr]`` is equivalent to ``OneOrMore(expr, stop_on=end_expr)`` + """ + stop_on_defined = False + stop_on = NoMatch() + if isinstance(key, slice): + key, stop_on = key.start, key.stop + if key is None: + key = ... + stop_on_defined = True + elif isinstance(key, tuple) and isinstance(key[-1], slice): + key, stop_on = (key[0], key[1].start), key[1].stop + stop_on_defined = True + # convert single arg keys to tuples + if isinstance(key, str_type): + key = (key,) try: - if isinstance(key, str_type): - key = (key,) iter(key) except TypeError: key = (key, key) if len(key) > 2: raise TypeError( - "only 1 or 2 index arguments supported ({}{})".format( - key[:5], "... [{}]".format(len(key)) if len(key) > 5 else "" - ) + f"only 1 or 2 index arguments supported ({key[:5]}{f'... [{len(key)}]' if len(key) > 5 else ''})" ) # clip to 2 elements ret = self * tuple(key[:2]) + ret = typing.cast(_MultipleMatch, ret) + + if stop_on_defined: + ret.stopOn(stop_on) + return ret - def __call__(self, name: str = None) -> "ParserElement": + def __call__(self, name: typing.Optional[str] = None) -> "ParserElement": """ Shortcut for :class:`set_results_name`, with ``list_all_matches=False``. If ``name`` is given with a trailing ``'*'`` character, then ``list_all_matches`` will be passed as ``True``. - If ``name` is omitted, same as calling :class:`copy`. + If ``name`` is omitted, same as calling :class:`copy`. Example:: @@ -1775,17 +1802,18 @@ def set_debug_actions( should have the signature ``fn(input_string: str, location: int, expression: ParserElement, exception: Exception, cache_hit: bool)`` """ self.debugActions = self.DebugActions( - start_action or _default_start_debug_action, - success_action or _default_success_debug_action, - exception_action or _default_exception_debug_action, + start_action or _default_start_debug_action, # type: ignore[truthy-function] + success_action or _default_success_debug_action, # type: ignore[truthy-function] + exception_action or _default_exception_debug_action, # type: ignore[truthy-function] ) self.debug = True return self - def set_debug(self, flag: bool = True) -> "ParserElement": + def set_debug(self, flag: bool = True, recurse: bool = False) -> "ParserElement": """ Enable display of debugging messages while doing pattern matching. Set ``flag`` to ``True`` to enable, ``False`` to disable. + Set ``recurse`` to ``True`` to set the debug flag on this expression and all sub-expressions. Example:: @@ -1819,6 +1847,11 @@ def set_debug(self, flag: bool = True) -> "ParserElement": which makes debugging and exception messages easier to understand - for instance, the default name created for the :class:`Word` expression without calling ``set_name`` is ``"W:(A-Za-z)"``. """ + if recurse: + for expr in self.visit_all(): + expr.set_debug(flag, recurse=False) + return self + if flag: self.set_debug_actions( _default_start_debug_action, @@ -1836,7 +1869,7 @@ def default_name(self) -> str: return self._defaultName @abstractmethod - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: """ Child classes must define this method, which defines how the ``default_name`` is set. """ @@ -1844,7 +1877,9 @@ def _generateDefaultName(self): def set_name(self, name: str) -> "ParserElement": """ Define name for this expression, makes debugging and exception messages clearer. + Example:: + Word(nums).parse_string("ABC") # -> Exception: Expected W:(0-9) (at char 0), (line:1, col:1) Word(nums).set_name("integer").parse_string("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ @@ -1870,7 +1905,7 @@ def streamline(self) -> "ParserElement": self._defaultName = None return self - def recurse(self) -> Sequence["ParserElement"]: + def recurse(self) -> List["ParserElement"]: return [] def _checkRecursion(self, parseElementList): @@ -1882,6 +1917,11 @@ def validate(self, validateTrace=None) -> None: """ Check defined expressions for valid structure, check for infinite recursive definitions. """ + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) self._checkRecursion([]) def parse_file( @@ -1899,8 +1939,10 @@ def parse_file( """ parseAll = parseAll or parse_all try: + file_or_filename = typing.cast(TextIO, file_or_filename) file_contents = file_or_filename.read() except AttributeError: + file_or_filename = typing.cast(str, file_or_filename) with open(file_or_filename, "r", encoding=encoding) as f: file_contents = f.read() try: @@ -1932,6 +1974,7 @@ def matches( inline microtests of sub expressions while building up larger parser. Parameters: + - ``test_string`` - to test against this expression for a match - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests @@ -1955,7 +1998,7 @@ def run_tests( full_dump: bool = True, print_results: bool = True, failure_tests: bool = False, - post_parse: Callable[[str, ParseResults], str] = None, + post_parse: typing.Optional[Callable[[str, ParseResults], str]] = None, file: typing.Optional[TextIO] = None, with_line_numbers: bool = False, *, @@ -1963,7 +2006,7 @@ def run_tests( fullDump: bool = True, printResults: bool = True, failureTests: bool = False, - postParse: Callable[[str, ParseResults], str] = None, + postParse: typing.Optional[Callable[[str, ParseResults], str]] = None, ) -> Tuple[bool, List[Tuple[str, Union[ParseResults, Exception]]]]: """ Execute the parse expression on a series of test strings, showing each @@ -1971,6 +2014,7 @@ def run_tests( run a parse expression against a list of sample strings. Parameters: + - ``tests`` - a list of separate test strings, or a multiline string of test strings - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests - ``comment`` - (default= ``'#'``) - expression for indicating embedded comments in the test @@ -2067,22 +2111,27 @@ def run_tests( failureTests = failureTests or failure_tests postParse = postParse or post_parse if isinstance(tests, str_type): + tests = typing.cast(str, tests) line_strip = type(tests).strip tests = [line_strip(test_line) for test_line in tests.rstrip().splitlines()] - if isinstance(comment, str_type): - comment = Literal(comment) + comment_specified = comment is not None + if comment_specified: + if isinstance(comment, str_type): + comment = typing.cast(str, comment) + comment = Literal(comment) + comment = typing.cast(ParserElement, comment) if file is None: file = sys.stdout print_ = file.write result: Union[ParseResults, Exception] - allResults = [] - comments = [] + allResults: List[Tuple[str, Union[ParseResults, Exception]]] = [] + comments: List[str] = [] success = True NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string) BOM = "\ufeff" for t in tests: - if comment is not None and comment.matches(t, False) or comments and not t: + if comment_specified and comment.matches(t, False) or comments and not t: comments.append( pyparsing_test.with_line_numbers(t) if with_line_numbers else t ) @@ -2107,7 +2156,7 @@ def run_tests( success = success and failureTests result = pe except Exception as exc: - out.append("FAIL-EXCEPTION: {}: {}".format(type(exc).__name__, exc)) + out.append(f"FAIL-EXCEPTION: {type(exc).__name__}: {exc}") if ParserElement.verbose_stacktrace: out.extend(traceback.format_tb(exc.__traceback__)) success = success and failureTests @@ -2127,9 +2176,7 @@ def run_tests( except Exception as e: out.append(result.dump(full=fullDump)) out.append( - "{} failed: {}: {}".format( - postParse.__name__, type(e).__name__, e - ) + f"{postParse.__name__} failed: {type(e).__name__}: {e}" ) else: out.append(result.dump(full=fullDump)) @@ -2148,19 +2195,28 @@ def create_diagram( vertical: int = 3, show_results_names: bool = False, show_groups: bool = False, + embed: bool = False, **kwargs, ) -> None: """ Create a railroad diagram for the parser. Parameters: - - output_html (str or file-like object) - output target for generated + + - ``output_html`` (str or file-like object) - output target for generated diagram HTML - - vertical (int) - threshold for formatting multiple alternatives vertically + - ``vertical`` (int) - threshold for formatting multiple alternatives vertically instead of horizontally (default=3) - - show_results_names - bool flag whether diagram should show annotations for + - ``show_results_names`` - bool flag whether diagram should show annotations for defined results names - - show_groups - bool flag whether groups should be highlighted with an unlabeled surrounding box + - ``show_groups`` - bool flag whether groups should be highlighted with an unlabeled surrounding box + - ``embed`` - bool flag whether generated HTML should omit , , and tags to embed + the resulting HTML in an enclosing HTML source + - ``head`` - str containing additional HTML to insert into the section of the generated code; + can be used to insert custom CSS styling + - ``body`` - str containing additional HTML to insert at the beginning of the section of the + generated code + Additional diagram-formatting keyword arguments can also be included; see railroad.Diagram class. """ @@ -2183,38 +2239,93 @@ def create_diagram( ) if isinstance(output_html, (str, Path)): with open(output_html, "w", encoding="utf-8") as diag_file: - diag_file.write(railroad_to_html(railroad)) + diag_file.write(railroad_to_html(railroad, embed=embed, **kwargs)) else: # we were passed a file-like object, just write to it - output_html.write(railroad_to_html(railroad)) - - setDefaultWhitespaceChars = set_default_whitespace_chars - inlineLiteralsUsing = inline_literals_using - setResultsName = set_results_name - setBreak = set_break - setParseAction = set_parse_action - addParseAction = add_parse_action - addCondition = add_condition - setFailAction = set_fail_action - tryParse = try_parse + output_html.write(railroad_to_html(railroad, embed=embed, **kwargs)) + + # Compatibility synonyms + # fmt: off + @staticmethod + @replaced_by_pep8(inline_literals_using) + def inlineLiteralsUsing(): ... + + @staticmethod + @replaced_by_pep8(set_default_whitespace_chars) + def setDefaultWhitespaceChars(): ... + + @replaced_by_pep8(set_results_name) + def setResultsName(self): ... + + @replaced_by_pep8(set_break) + def setBreak(self): ... + + @replaced_by_pep8(set_parse_action) + def setParseAction(self): ... + + @replaced_by_pep8(add_parse_action) + def addParseAction(self): ... + + @replaced_by_pep8(add_condition) + def addCondition(self): ... + + @replaced_by_pep8(set_fail_action) + def setFailAction(self): ... + + @replaced_by_pep8(try_parse) + def tryParse(self): ... + + @staticmethod + @replaced_by_pep8(enable_left_recursion) + def enableLeftRecursion(): ... + + @staticmethod + @replaced_by_pep8(enable_packrat) + def enablePackrat(): ... + + @replaced_by_pep8(parse_string) + def parseString(self): ... + + @replaced_by_pep8(scan_string) + def scanString(self): ... + + @replaced_by_pep8(transform_string) + def transformString(self): ... + + @replaced_by_pep8(search_string) + def searchString(self): ... + + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... + + @replaced_by_pep8(set_whitespace_chars) + def setWhitespaceChars(self): ... + + @replaced_by_pep8(parse_with_tabs) + def parseWithTabs(self): ... + + @replaced_by_pep8(set_debug_actions) + def setDebugActions(self): ... + + @replaced_by_pep8(set_debug) + def setDebug(self): ... + + @replaced_by_pep8(set_name) + def setName(self): ... + + @replaced_by_pep8(parse_file) + def parseFile(self): ... + + @replaced_by_pep8(run_tests) + def runTests(self): ... + canParseNext = can_parse_next resetCache = reset_cache - enableLeftRecursion = enable_left_recursion - enablePackrat = enable_packrat - parseString = parse_string - scanString = scan_string - searchString = search_string - transformString = transform_string - setWhitespaceChars = set_whitespace_chars - parseWithTabs = parse_with_tabs - setDebugActions = set_debug_actions - setDebug = set_debug defaultName = default_name - setName = set_name - parseFile = parse_file - runTests = run_tests - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + # fmt: on class _PendingSkip(ParserElement): @@ -2225,7 +2336,7 @@ def __init__(self, expr: ParserElement, must_skip: bool = False): self.anchor = expr self.must_skip = must_skip - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return str(self.anchor + Empty()).replace("Empty", "...") def __add__(self, other) -> "ParserElement": @@ -2266,21 +2377,10 @@ class Token(ParserElement): def __init__(self): super().__init__(savelist=False) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return type(self).__name__ -class Empty(Token): - """ - An empty token, will always match. - """ - - def __init__(self): - super().__init__() - self.mayReturnEmpty = True - self.mayIndexError = False - - class NoMatch(Token): """ A token that will never match. @@ -2312,25 +2412,33 @@ class Literal(Token): use :class:`Keyword` or :class:`CaselessKeyword`. """ + def __new__(cls, match_string: str = "", *, matchString: str = ""): + # Performance tuning: select a subclass with optimized parseImpl + if cls is Literal: + match_string = matchString or match_string + if not match_string: + return super().__new__(Empty) + if len(match_string) == 1: + return super().__new__(_SingleCharLiteral) + + # Default behavior + return super().__new__(cls) + + # Needed to make copy.copy() work correctly if we customize __new__ + def __getnewargs__(self): + return (self.match,) + def __init__(self, match_string: str = "", *, matchString: str = ""): super().__init__() match_string = matchString or match_string self.match = match_string self.matchLen = len(match_string) - try: - self.firstMatchChar = match_string[0] - except IndexError: - raise ValueError("null string passed to Literal; use Empty() instead") + self.firstMatchChar = match_string[:1] self.errmsg = "Expected " + self.name self.mayReturnEmpty = False self.mayIndexError = False - # Performance tuning: modify __class__ to select - # a parseImpl optimized for single-character check - if self.matchLen == 1 and type(self) is Literal: - self.__class__ = _SingleCharLiteral - - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return repr(self.match) def parseImpl(self, instring, loc, doActions=True): @@ -2341,6 +2449,23 @@ def parseImpl(self, instring, loc, doActions=True): raise ParseException(instring, loc, self.errmsg, self) +class Empty(Literal): + """ + An empty token, will always match. + """ + + def __init__(self, match_string="", *, matchString=""): + super().__init__("") + self.mayReturnEmpty = True + self.mayIndexError = False + + def _generateDefaultName(self) -> str: + return "Empty" + + def parseImpl(self, instring, loc, doActions=True): + return loc, [] + + class _SingleCharLiteral(Literal): def parseImpl(self, instring, loc, doActions=True): if instring[loc] == self.firstMatchChar: @@ -2354,8 +2479,8 @@ def parseImpl(self, instring, loc, doActions=True): class Keyword(Token): """ Token to exactly match a specified string as a keyword, that is, - it must be immediately followed by a non-keyword character. Compare - with :class:`Literal`: + it must be immediately preceded and followed by whitespace or + non-keyword characters. Compare with :class:`Literal`: - ``Literal("if")`` will match the leading ``'if'`` in ``'ifAndOnlyIf'``. @@ -2365,7 +2490,7 @@ class Keyword(Token): Accepts two optional constructor arguments in addition to the keyword string: - - ``identChars`` is a string of characters that would be valid + - ``ident_chars`` is a string of characters that would be valid identifier characters, defaulting to all alphanumerics + "_" and "$" - ``caseless`` allows case-insensitive matching, default is ``False``. @@ -2400,7 +2525,7 @@ def __init__( self.firstMatchChar = match_string[0] except IndexError: raise ValueError("null string passed to Keyword; use Empty() instead") - self.errmsg = "Expected {} {}".format(type(self).__name__, self.name) + self.errmsg = f"Expected {type(self).__name__} {self.name}" self.mayReturnEmpty = False self.mayIndexError = False self.caseless = caseless @@ -2409,7 +2534,7 @@ def __init__( identChars = identChars.upper() self.identChars = set(identChars) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return repr(self.match) def parseImpl(self, instring, loc, doActions=True): @@ -2559,7 +2684,7 @@ class CloseMatch(Token): def __init__( self, match_string: str, - max_mismatches: int = None, + max_mismatches: typing.Optional[int] = None, *, maxMismatches: int = 1, caseless=False, @@ -2568,15 +2693,13 @@ def __init__( super().__init__() self.match_string = match_string self.maxMismatches = maxMismatches - self.errmsg = "Expected {!r} (with up to {} mismatches)".format( - self.match_string, self.maxMismatches - ) + self.errmsg = f"Expected {self.match_string!r} (with up to {self.maxMismatches} mismatches)" self.caseless = caseless self.mayIndexError = False self.mayReturnEmpty = False - def _generateDefaultName(self): - return "{}:{!r}".format(type(self).__name__, self.match_string) + def _generateDefaultName(self) -> str: + return f"{type(self).__name__}:{self.match_string!r}" def parseImpl(self, instring, loc, doActions=True): start = loc @@ -2612,7 +2735,9 @@ def parseImpl(self, instring, loc, doActions=True): class Word(Token): """Token for matching words composed of allowed character sets. + Parameters: + - ``init_chars`` - string of all characters that should be used to match as a word; "ABC" will match "AAA", "ABAB", "CBAC", etc.; if ``body_chars`` is also specified, then this is the string of @@ -2697,26 +2822,24 @@ def __init__( super().__init__() if not initChars: raise ValueError( - "invalid {}, initChars cannot be empty string".format( - type(self).__name__ - ) + f"invalid {type(self).__name__}, initChars cannot be empty string" ) - initChars = set(initChars) - self.initChars = initChars + initChars_set = set(initChars) if excludeChars: - excludeChars = set(excludeChars) - initChars -= excludeChars + excludeChars_set = set(excludeChars) + initChars_set -= excludeChars_set if bodyChars: - bodyChars = set(bodyChars) - excludeChars - self.initCharsOrig = "".join(sorted(initChars)) + bodyChars = "".join(set(bodyChars) - excludeChars_set) + self.initChars = initChars_set + self.initCharsOrig = "".join(sorted(initChars_set)) if bodyChars: - self.bodyCharsOrig = "".join(sorted(bodyChars)) self.bodyChars = set(bodyChars) + self.bodyCharsOrig = "".join(sorted(bodyChars)) else: - self.bodyCharsOrig = "".join(sorted(initChars)) - self.bodyChars = set(initChars) + self.bodyChars = initChars_set + self.bodyCharsOrig = self.initCharsOrig self.maxSpecified = max > 0 @@ -2725,6 +2848,11 @@ def __init__( "cannot specify a minimum length < 1; use Opt(Word()) if zero-length word is permitted" ) + if self.maxSpecified and min > max: + raise ValueError( + f"invalid args, if min and max both specified min must be <= max (min={min}, max={max})" + ) + self.minLen = min if max > 0: @@ -2733,62 +2861,66 @@ def __init__( self.maxLen = _MAX_INT if exact > 0: + min = max = exact self.maxLen = exact self.minLen = exact self.errmsg = "Expected " + self.name self.mayIndexError = False self.asKeyword = asKeyword + if self.asKeyword: + self.errmsg += " as a keyword" # see if we can make a regex for this Word - if " " not in self.initChars | self.bodyChars and (min == 1 and exact == 0): + if " " not in (self.initChars | self.bodyChars): + if len(self.initChars) == 1: + re_leading_fragment = re.escape(self.initCharsOrig) + else: + re_leading_fragment = f"[{_collapse_string_to_ranges(self.initChars)}]" + if self.bodyChars == self.initChars: if max == 0: repeat = "+" elif max == 1: repeat = "" else: - repeat = "{{{},{}}}".format( - self.minLen, "" if self.maxLen == _MAX_INT else self.maxLen - ) - self.reString = "[{}]{}".format( - _collapse_string_to_ranges(self.initChars), - repeat, - ) - elif len(self.initChars) == 1: - if max == 0: - repeat = "*" - else: - repeat = "{{0,{}}}".format(max - 1) - self.reString = "{}[{}]{}".format( - re.escape(self.initCharsOrig), - _collapse_string_to_ranges(self.bodyChars), - repeat, - ) + if self.minLen != self.maxLen: + repeat = f"{{{self.minLen},{'' if self.maxLen == _MAX_INT else self.maxLen}}}" + else: + repeat = f"{{{self.minLen}}}" + self.reString = f"{re_leading_fragment}{repeat}" else: - if max == 0: - repeat = "*" - elif max == 2: + if max == 1: + re_body_fragment = "" repeat = "" else: - repeat = "{{0,{}}}".format(max - 1) - self.reString = "[{}][{}]{}".format( - _collapse_string_to_ranges(self.initChars), - _collapse_string_to_ranges(self.bodyChars), - repeat, + re_body_fragment = f"[{_collapse_string_to_ranges(self.bodyChars)}]" + if max == 0: + repeat = "*" + elif max == 2: + repeat = "?" if min <= 1 else "" + else: + if min != max: + repeat = f"{{{min - 1 if min > 0 else 0},{max - 1}}}" + else: + repeat = f"{{{min - 1 if min > 0 else 0}}}" + + self.reString = ( + f"{re_leading_fragment}" f"{re_body_fragment}" f"{repeat}" ) + if self.asKeyword: - self.reString = r"\b" + self.reString + r"\b" + self.reString = rf"\b{self.reString}\b" try: self.re = re.compile(self.reString) except re.error: - self.re = None + self.re = None # type: ignore[assignment] else: self.re_match = self.re.match - self.__class__ = _WordRegex + self.parseImpl = self.parseImpl_regex # type: ignore[assignment] - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: def charsAsStr(s): max_repr_len = 16 s = _collapse_string_to_ranges(s, re_escape=False) @@ -2798,11 +2930,9 @@ def charsAsStr(s): return s if self.initChars != self.bodyChars: - base = "W:({}, {})".format( - charsAsStr(self.initChars), charsAsStr(self.bodyChars) - ) + base = f"W:({charsAsStr(self.initChars)}, {charsAsStr(self.bodyChars)})" else: - base = "W:({})".format(charsAsStr(self.initChars)) + base = f"W:({charsAsStr(self.initChars)})" # add length specification if self.minLen > 1 or self.maxLen != _MAX_INT: @@ -2810,11 +2940,11 @@ def charsAsStr(s): if self.minLen == 1: return base[2:] else: - return base + "{{{}}}".format(self.minLen) + return base + f"{{{self.minLen}}}" elif self.maxLen == _MAX_INT: - return base + "{{{},...}}".format(self.minLen) + return base + f"{{{self.minLen},...}}" else: - return base + "{{{},{}}}".format(self.minLen, self.maxLen) + return base + f"{{{self.minLen},{self.maxLen}}}" return base def parseImpl(self, instring, loc, doActions=True): @@ -2849,9 +2979,7 @@ def parseImpl(self, instring, loc, doActions=True): return loc, instring[start:loc] - -class _WordRegex(Word): - def parseImpl(self, instring, loc, doActions=True): + def parseImpl_regex(self, instring, loc, doActions=True): result = self.re_match(instring, loc) if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -2860,7 +2988,7 @@ def parseImpl(self, instring, loc, doActions=True): return loc, result.group() -class Char(_WordRegex): +class Char(Word): """A short-cut class for defining :class:`Word` ``(characters, exact=1)``, when defining a match of any single character in a string of characters. @@ -2878,13 +3006,8 @@ def __init__( asKeyword = asKeyword or as_keyword excludeChars = excludeChars or exclude_chars super().__init__( - charset, exact=1, asKeyword=asKeyword, excludeChars=excludeChars + charset, exact=1, as_keyword=asKeyword, exclude_chars=excludeChars ) - self.reString = "[{}]".format(_collapse_string_to_ranges(self.initChars)) - if asKeyword: - self.reString = r"\b{}\b".format(self.reString) - self.re = re.compile(self.reString) - self.re_match = self.re.match class Regex(Token): @@ -2954,9 +3077,9 @@ def __init__( self.asGroupList = asGroupList self.asMatch = asMatch if self.asGroupList: - self.parseImpl = self.parseImplAsGroupList + self.parseImpl = self.parseImplAsGroupList # type: ignore [assignment] if self.asMatch: - self.parseImpl = self.parseImplAsMatch + self.parseImpl = self.parseImplAsMatch # type: ignore [assignment] @cached_property def re(self): @@ -2966,9 +3089,7 @@ def re(self): try: return re.compile(self.pattern, self.flags) except re.error: - raise ValueError( - "invalid pattern ({!r}) passed to Regex".format(self.pattern) - ) + raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex") @cached_property def re_match(self): @@ -2978,7 +3099,7 @@ def re_match(self): def mayReturnEmpty(self): return self.re_match("") is not None - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "Re:({})".format(repr(self.pattern).replace("\\\\", "\\")) def parseImpl(self, instring, loc, doActions=True): @@ -3024,10 +3145,12 @@ def sub(self, repl: str) -> ParserElement: # prints "

main title

" """ if self.asGroupList: - raise TypeError("cannot use sub() with Regex(asGroupList=True)") + raise TypeError("cannot use sub() with Regex(as_group_list=True)") if self.asMatch and callable(repl): - raise TypeError("cannot use sub() with a callable with Regex(asMatch=True)") + raise TypeError( + "cannot use sub() with a callable with Regex(as_match=True)" + ) if self.asMatch: @@ -3081,7 +3204,7 @@ class QuotedString(Token): [['This is the "quote"']] [['This is the quote with "embedded" quotes']] """ - ws_map = ((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r")) + ws_map = dict(((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r"))) def __init__( self, @@ -3120,57 +3243,54 @@ def __init__( else: endQuoteChar = endQuoteChar.strip() if not endQuoteChar: - raise ValueError("endQuoteChar cannot be the empty string") - - self.quoteChar = quote_char - self.quoteCharLen = len(quote_char) - self.firstQuoteChar = quote_char[0] - self.endQuoteChar = endQuoteChar - self.endQuoteCharLen = len(endQuoteChar) - self.escChar = escChar - self.escQuote = escQuote - self.unquoteResults = unquoteResults - self.convertWhitespaceEscapes = convertWhitespaceEscapes + raise ValueError("end_quote_char cannot be the empty string") + + self.quoteChar: str = quote_char + self.quoteCharLen: int = len(quote_char) + self.firstQuoteChar: str = quote_char[0] + self.endQuoteChar: str = endQuoteChar + self.endQuoteCharLen: int = len(endQuoteChar) + self.escChar: str = escChar or "" + self.escQuote: str = escQuote or "" + self.unquoteResults: bool = unquoteResults + self.convertWhitespaceEscapes: bool = convertWhitespaceEscapes + self.multiline = multiline sep = "" inner_pattern = "" if escQuote: - inner_pattern += r"{}(?:{})".format(sep, re.escape(escQuote)) + inner_pattern += rf"{sep}(?:{re.escape(escQuote)})" sep = "|" if escChar: - inner_pattern += r"{}(?:{}.)".format(sep, re.escape(escChar)) + inner_pattern += rf"{sep}(?:{re.escape(escChar)}.)" sep = "|" - self.escCharReplacePattern = re.escape(self.escChar) + "(.)" + self.escCharReplacePattern = re.escape(escChar) + "(.)" if len(self.endQuoteChar) > 1: inner_pattern += ( - "{}(?:".format(sep) + f"{sep}(?:" + "|".join( - "(?:{}(?!{}))".format( - re.escape(self.endQuoteChar[:i]), - re.escape(self.endQuoteChar[i:]), - ) + f"(?:{re.escape(self.endQuoteChar[:i])}(?!{re.escape(self.endQuoteChar[i:])}))" for i in range(len(self.endQuoteChar) - 1, 0, -1) ) + ")" ) sep = "|" + self.flags = re.RegexFlag(0) + if multiline: self.flags = re.MULTILINE | re.DOTALL - inner_pattern += r"{}(?:[^{}{}])".format( - sep, - _escape_regex_range_chars(self.endQuoteChar[0]), - (_escape_regex_range_chars(escChar) if escChar is not None else ""), + inner_pattern += ( + rf"{sep}(?:[^{_escape_regex_range_chars(self.endQuoteChar[0])}" + rf"{(_escape_regex_range_chars(escChar) if escChar is not None else '')}])" ) else: - self.flags = 0 - inner_pattern += r"{}(?:[^{}\n\r{}])".format( - sep, - _escape_regex_range_chars(self.endQuoteChar[0]), - (_escape_regex_range_chars(escChar) if escChar is not None else ""), + inner_pattern += ( + rf"{sep}(?:[^{_escape_regex_range_chars(self.endQuoteChar[0])}\n\r" + rf"{(_escape_regex_range_chars(escChar) if escChar is not None else '')}])" ) self.pattern = "".join( @@ -3183,26 +3303,33 @@ def __init__( ] ) + if self.unquoteResults: + if self.convertWhitespaceEscapes: + self.unquote_scan_re = re.compile( + rf"({'|'.join(re.escape(k) for k in self.ws_map)})|({re.escape(self.escChar)}.)|(\n|.)", + flags=self.flags, + ) + else: + self.unquote_scan_re = re.compile( + rf"({re.escape(self.escChar)}.)|(\n|.)", flags=self.flags + ) + try: self.re = re.compile(self.pattern, self.flags) self.reString = self.pattern self.re_match = self.re.match except re.error: - raise ValueError( - "invalid pattern {!r} passed to Regex".format(self.pattern) - ) + raise ValueError(f"invalid pattern {self.pattern!r} passed to Regex") self.errmsg = "Expected " + self.name self.mayIndexError = False self.mayReturnEmpty = True - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: if self.quoteChar == self.endQuoteChar and isinstance(self.quoteChar, str_type): - return "string enclosed in {!r}".format(self.quoteChar) + return f"string enclosed in {self.quoteChar!r}" - return "quoted string, starting with {} ending with {}".format( - self.quoteChar, self.endQuoteChar - ) + return f"quoted string, starting with {self.quoteChar} ending with {self.endQuoteChar}" def parseImpl(self, instring, loc, doActions=True): result = ( @@ -3217,19 +3344,24 @@ def parseImpl(self, instring, loc, doActions=True): ret = result.group() if self.unquoteResults: - # strip off quotes ret = ret[self.quoteCharLen : -self.endQuoteCharLen] if isinstance(ret, str_type): - # replace escaped whitespace - if "\\" in ret and self.convertWhitespaceEscapes: - for wslit, wschar in self.ws_map: - ret = ret.replace(wslit, wschar) - - # replace escaped characters - if self.escChar: - ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret) + if self.convertWhitespaceEscapes: + ret = "".join( + self.ws_map[match.group(1)] + if match.group(1) + else match.group(2)[-1] + if match.group(2) + else match.group(3) + for match in self.unquote_scan_re.finditer(ret) + ) + else: + ret = "".join( + match.group(1)[-1] if match.group(1) else match.group(2) + for match in self.unquote_scan_re.finditer(ret) + ) # replace escaped quotes if self.escQuote: @@ -3252,7 +3384,7 @@ class CharsNotIn(Token): # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') - print(delimited_list(csv_value).parse_string("dkls,lsdkjf,s12 34,@!#,213")) + print(DelimitedList(csv_value).parse_string("dkls,lsdkjf,s12 34,@!#,213")) prints:: @@ -3294,12 +3426,12 @@ def __init__( self.mayReturnEmpty = self.minLen == 0 self.mayIndexError = False - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: not_chars_str = _collapse_string_to_ranges(self.notChars) if len(not_chars_str) > 16: - return "!W:({}...)".format(self.notChars[: 16 - 3]) + return f"!W:({self.notChars[: 16 - 3]}...)" else: - return "!W:({})".format(self.notChars) + return f"!W:({self.notChars})" def parseImpl(self, instring, loc, doActions=True): notchars = self.notCharsSet @@ -3376,7 +3508,7 @@ def __init__(self, ws: str = " \t\r\n", min: int = 1, max: int = 0, exact: int = self.maxLen = exact self.minLen = exact - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "".join(White.whiteStrs[c] for c in self.matchWhite) def parseImpl(self, instring, loc, doActions=True): @@ -3411,7 +3543,7 @@ def __init__(self, colno: int): super().__init__() self.col = colno - def preParse(self, instring, loc): + def preParse(self, instring: str, loc: int) -> int: if col(loc, instring) != self.col: instrlen = len(instring) if self.ignoreExprs: @@ -3446,7 +3578,7 @@ class LineStart(PositionToken): B AAA and definitely not this one ''' - for t in (LineStart() + 'AAA' + restOfLine).search_string(test): + for t in (LineStart() + 'AAA' + rest_of_line).search_string(test): print(t) prints:: @@ -3464,7 +3596,7 @@ def __init__(self): self.skipper = Empty().set_whitespace_chars(self.whiteChars) self.errmsg = "Expected start of line" - def preParse(self, instring, loc): + def preParse(self, instring: str, loc: int) -> int: if loc == 0: return loc else: @@ -3624,7 +3756,7 @@ def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = False self.exprs = [exprs] self.callPreparse = False - def recurse(self) -> Sequence[ParserElement]: + def recurse(self) -> List[ParserElement]: return self.exprs[:] def append(self, other) -> ParserElement: @@ -3669,8 +3801,8 @@ def ignore(self, other) -> ParserElement: e.ignore(self.ignoreExprs[-1]) return self - def _generateDefaultName(self): - return "{}:({})".format(self.__class__.__name__, str(self.exprs)) + def _generateDefaultName(self) -> str: + return f"{self.__class__.__name__}:({str(self.exprs)})" def streamline(self) -> ParserElement: if self.streamlined: @@ -3714,6 +3846,11 @@ def streamline(self) -> ParserElement: return self def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) tmp = (validateTrace if validateTrace is not None else [])[:] + [self] for e in self.exprs: e.validate(tmp) @@ -3721,6 +3858,7 @@ def validate(self, validateTrace=None) -> None: def copy(self) -> ParserElement: ret = super().copy() + ret = typing.cast(ParseExpression, ret) ret.exprs = [e.copy() for e in self.exprs] return ret @@ -3750,8 +3888,14 @@ def _setResultsName(self, name, listAllMatches=False): return super()._setResultsName(name, listAllMatches) - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + # Compatibility synonyms + # fmt: off + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... + + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + # fmt: on class And(ParseExpression): @@ -3777,7 +3921,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.leave_whitespace() - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "-" def __init__( @@ -3789,7 +3933,9 @@ def __init__( for i, expr in enumerate(exprs): if expr is Ellipsis: if i < len(exprs) - 1: - skipto_arg: ParserElement = (Empty() + exprs[i + 1]).exprs[-1] + skipto_arg: ParserElement = typing.cast( + ParseExpression, (Empty() + exprs[i + 1]) + ).exprs[-1] tmp.append(SkipTo(skipto_arg)("_skipped*")) else: raise Exception( @@ -3822,8 +3968,9 @@ def streamline(self) -> ParserElement: and isinstance(e.exprs[-1], _PendingSkip) for e in self.exprs[:-1] ): + deleted_expr_marker = NoMatch() for i, e in enumerate(self.exprs[:-1]): - if e is None: + if e is deleted_expr_marker: continue if ( isinstance(e, ParseExpression) @@ -3831,17 +3978,19 @@ def streamline(self) -> ParserElement: and isinstance(e.exprs[-1], _PendingSkip) ): e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1] - self.exprs[i + 1] = None - self.exprs = [e for e in self.exprs if e is not None] + self.exprs[i + 1] = deleted_expr_marker + self.exprs = [e for e in self.exprs if e is not deleted_expr_marker] super().streamline() # link any IndentedBlocks to the prior expression + prev: ParserElement + cur: ParserElement for prev, cur in zip(self.exprs, self.exprs[1:]): # traverse cur or any first embedded expr of cur looking for an IndentedBlock # (but watch out for recursive grammar) seen = set() - while cur: + while True: if id(cur) in seen: break seen.add(id(cur)) @@ -3853,7 +4002,10 @@ def streamline(self) -> ParserElement: ) break subs = cur.recurse() - cur = next(iter(subs), None) + next_first = next(iter(subs), None) + if next_first is None: + break + cur = typing.cast(ParserElement, next_first) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) return self @@ -3884,13 +4036,14 @@ def parseImpl(self, instring, loc, doActions=True): ) else: loc, exprtokens = e._parse(instring, loc, doActions) - if exprtokens or exprtokens.haskeys(): - resultlist += exprtokens + resultlist += exprtokens return loc, resultlist def __iadd__(self, other): if isinstance(other, str_type): other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented return self.append(other) # And([self, other]) def _checkRecursion(self, parseElementList): @@ -3900,7 +4053,7 @@ def _checkRecursion(self, parseElementList): if not e.mayReturnEmpty: break - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: inner = " ".join(str(e) for e in self.exprs) # strip off redundant inner {}'s while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}": @@ -3958,7 +4111,7 @@ def parseImpl(self, instring, loc, doActions=True): loc2 = e.try_parse(instring, loc, raise_fatal=True) except ParseFatalException as pfe: pfe.__traceback__ = None - pfe.parserElement = e + pfe.parser_element = e fatals.append(pfe) maxException = None maxExcLoc = -1 @@ -4016,12 +4169,15 @@ def parseImpl(self, instring, loc, doActions=True): if len(fatals) > 1: fatals.sort(key=lambda e: -e.loc) if fatals[0].loc == fatals[1].loc: - fatals.sort(key=lambda e: (-e.loc, -len(str(e.parserElement)))) + fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element)))) max_fatal = fatals[0] raise max_fatal if maxException is not None: - maxException.msg = self.errmsg + # infer from this check that all alternatives failed at the current position + # so emit this collective error message instead of any single error message + if maxExcLoc == loc: + maxException.msg = self.errmsg raise maxException else: raise ParseException( @@ -4031,9 +4187,11 @@ def parseImpl(self, instring, loc, doActions=True): def __ixor__(self, other): if isinstance(other, str_type): other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented return self.append(other) # Or([self, other]) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + " ^ ".join(str(e) for e in self.exprs) + "}" def _setResultsName(self, name, listAllMatches=False): @@ -4118,7 +4276,7 @@ def parseImpl(self, instring, loc, doActions=True): ) except ParseFatalException as pfe: pfe.__traceback__ = None - pfe.parserElement = e + pfe.parser_element = e raise except ParseException as err: if err.loc > maxExcLoc: @@ -4132,7 +4290,10 @@ def parseImpl(self, instring, loc, doActions=True): maxExcLoc = len(instring) if maxException is not None: - maxException.msg = self.errmsg + # infer from this check that all alternatives failed at the current position + # so emit this collective error message instead of any individual error message + if maxExcLoc == loc: + maxException.msg = self.errmsg raise maxException else: raise ParseException( @@ -4142,9 +4303,11 @@ def parseImpl(self, instring, loc, doActions=True): def __ior__(self, other): if isinstance(other, str_type): other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented return self.append(other) # MatchFirst([self, other]) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + " | ".join(str(e) for e in self.exprs) + "}" def _setResultsName(self, name, listAllMatches=False): @@ -4242,6 +4405,13 @@ def __init__(self, exprs: typing.Iterable[ParserElement], savelist: bool = True) self.initExprGroups = True self.saveAsList = True + def __iand__(self, other): + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return self.append(other) # Each([self, other]) + def streamline(self) -> ParserElement: super().streamline() if self.exprs: @@ -4296,7 +4466,7 @@ def parseImpl(self, instring, loc, doActions=True): tmpLoc = e.try_parse(instring, tmpLoc, raise_fatal=True) except ParseFatalException as pfe: pfe.__traceback__ = None - pfe.parserElement = e + pfe.parser_element = e fatals.append(pfe) failed.append(e) except ParseException: @@ -4315,7 +4485,7 @@ def parseImpl(self, instring, loc, doActions=True): if len(fatals) > 1: fatals.sort(key=lambda e: -e.loc) if fatals[0].loc == fatals[1].loc: - fatals.sort(key=lambda e: (-e.loc, -len(str(e.parserElement)))) + fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element)))) max_fatal = fatals[0] raise max_fatal @@ -4324,7 +4494,7 @@ def parseImpl(self, instring, loc, doActions=True): raise ParseException( instring, loc, - "Missing one or more required elements ({})".format(missing), + f"Missing one or more required elements ({missing})", ) # add any unmatched Opts, in case they have default values defined @@ -4337,7 +4507,7 @@ def parseImpl(self, instring, loc, doActions=True): return loc, total_results - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + " & ".join(str(e) for e in self.exprs) + "}" @@ -4349,12 +4519,14 @@ class ParseElementEnhance(ParserElement): def __init__(self, expr: Union[ParserElement, str], savelist: bool = False): super().__init__(savelist) if isinstance(expr, str_type): + expr_str = typing.cast(str, expr) if issubclass(self._literalStringClass, Token): - expr = self._literalStringClass(expr) + expr = self._literalStringClass(expr_str) # type: ignore[call-arg] elif issubclass(type(self), self._literalStringClass): - expr = Literal(expr) + expr = Literal(expr_str) else: - expr = self._literalStringClass(Literal(expr)) + expr = self._literalStringClass(Literal(expr_str)) # type: ignore[assignment, call-arg] + expr = typing.cast(ParserElement, expr) self.expr = expr if expr is not None: self.mayIndexError = expr.mayIndexError @@ -4367,12 +4539,16 @@ def __init__(self, expr: Union[ParserElement, str], savelist: bool = False): self.callPreparse = expr.callPreparse self.ignoreExprs.extend(expr.ignoreExprs) - def recurse(self) -> Sequence[ParserElement]: + def recurse(self) -> List[ParserElement]: return [self.expr] if self.expr is not None else [] def parseImpl(self, instring, loc, doActions=True): if self.expr is not None: - return self.expr._parse(instring, loc, doActions, callPreParse=False) + try: + return self.expr._parse(instring, loc, doActions, callPreParse=False) + except ParseBaseException as pbe: + pbe.msg = self.errmsg + raise else: raise ParseException(instring, loc, "No expression defined", self) @@ -4380,8 +4556,8 @@ def leave_whitespace(self, recursive: bool = True) -> ParserElement: super().leave_whitespace(recursive) if recursive: - self.expr = self.expr.copy() if self.expr is not None: + self.expr = self.expr.copy() self.expr.leave_whitespace(recursive) return self @@ -4389,8 +4565,8 @@ def ignore_whitespace(self, recursive: bool = True) -> ParserElement: super().ignore_whitespace(recursive) if recursive: - self.expr = self.expr.copy() if self.expr is not None: + self.expr = self.expr.copy() self.expr.ignore_whitespace(recursive) return self @@ -4420,6 +4596,11 @@ def _checkRecursion(self, parseElementList): self.expr._checkRecursion(subRecCheckList) def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) if validateTrace is None: validateTrace = [] tmp = validateTrace[:] + [self] @@ -4427,11 +4608,17 @@ def validate(self, validateTrace=None) -> None: self.expr.validate(tmp) self._checkRecursion([]) - def _generateDefaultName(self): - return "{}:({})".format(self.__class__.__name__, str(self.expr)) + def _generateDefaultName(self) -> str: + return f"{self.__class__.__name__}:({str(self.expr)})" + + # Compatibility synonyms + # fmt: off + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + # fmt: on class IndentedBlock(ParseElementEnhance): @@ -4443,13 +4630,13 @@ class IndentedBlock(ParseElementEnhance): class _Indent(Empty): def __init__(self, ref_col: int): super().__init__() - self.errmsg = "expected indent at column {}".format(ref_col) + self.errmsg = f"expected indent at column {ref_col}" self.add_condition(lambda s, l, t: col(l, s) == ref_col) class _IndentGreater(Empty): def __init__(self, ref_col: int): super().__init__() - self.errmsg = "expected indent at column greater than {}".format(ref_col) + self.errmsg = f"expected indent at column greater than {ref_col}" self.add_condition(lambda s, l, t: col(l, s) > ref_col) def __init__( @@ -4469,7 +4656,7 @@ def parseImpl(self, instring, loc, doActions=True): # see if self.expr matches at the current location - if not it will raise an exception # and no further work is necessary - self.expr.try_parse(instring, anchor_loc, doActions) + self.expr.try_parse(instring, anchor_loc, do_actions=doActions) indent_col = col(anchor_loc, instring) peer_detect_expr = self._Indent(indent_col) @@ -4532,7 +4719,7 @@ class AtLineStart(ParseElementEnhance): B AAA and definitely not this one ''' - for t in (AtLineStart('AAA') + restOfLine).search_string(test): + for t in (AtLineStart('AAA') + rest_of_line).search_string(test): print(t) prints:: @@ -4598,9 +4785,9 @@ class PrecededBy(ParseElementEnhance): Parameters: - - expr - expression that must match prior to the current parse + - ``expr`` - expression that must match prior to the current parse location - - retreat - (default= ``None``) - (int) maximum number of characters + - ``retreat`` - (default= ``None``) - (int) maximum number of characters to lookbehind prior to the current parse location If the lookbehind expression is a string, :class:`Literal`, @@ -4627,6 +4814,7 @@ def __init__( self.mayIndexError = False self.exact = False if isinstance(expr, str_type): + expr = typing.cast(str, expr) retreat = len(expr) self.exact = True elif isinstance(expr, (Literal, Keyword)): @@ -4746,18 +4934,18 @@ def __init__(self, expr: Union[ParserElement, str]): self.errmsg = "Found unwanted token, " + str(self.expr) def parseImpl(self, instring, loc, doActions=True): - if self.expr.can_parse_next(instring, loc): + if self.expr.can_parse_next(instring, loc, do_actions=doActions): raise ParseException(instring, loc, self.errmsg, self) return loc, [] - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "~{" + str(self.expr) + "}" class _MultipleMatch(ParseElementEnhance): def __init__( self, - expr: ParserElement, + expr: Union[str, ParserElement], stop_on: typing.Optional[Union[ParserElement, str]] = None, *, stopOn: typing.Optional[Union[ParserElement, str]] = None, @@ -4781,7 +4969,7 @@ def parseImpl(self, instring, loc, doActions=True): self_skip_ignorables = self._skipIgnorables check_ender = self.not_ender is not None if check_ender: - try_not_ender = self.not_ender.tryParse + try_not_ender = self.not_ender.try_parse # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) @@ -4798,8 +4986,7 @@ def parseImpl(self, instring, loc, doActions=True): else: preloc = loc loc, tmptokens = self_expr_parse(instring, preloc, doActions) - if tmptokens or tmptokens.haskeys(): - tokens += tmptokens + tokens += tmptokens except (ParseException, IndexError): pass @@ -4837,10 +5024,11 @@ class OneOrMore(_MultipleMatch): Repetition of one or more of the given expression. Parameters: - - expr - expression that must match one or more times - - stop_on - (default= ``None``) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + + - ``expr`` - expression that must match one or more times + - ``stop_on`` - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: @@ -4859,7 +5047,7 @@ class OneOrMore(_MultipleMatch): (attr_expr * (1,)).parse_string(text).pprint() """ - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + str(self.expr) + "}..." @@ -4868,6 +5056,7 @@ class ZeroOrMore(_MultipleMatch): Optional repetition of zero or more of the given expression. Parameters: + - ``expr`` - expression that must match zero or more times - ``stop_on`` - expression for a terminating sentinel (only required if the sentinel would ordinarily match the repetition @@ -4878,7 +5067,7 @@ class ZeroOrMore(_MultipleMatch): def __init__( self, - expr: ParserElement, + expr: Union[str, ParserElement], stop_on: typing.Optional[Union[ParserElement, str]] = None, *, stopOn: typing.Optional[Union[ParserElement, str]] = None, @@ -4892,10 +5081,75 @@ def parseImpl(self, instring, loc, doActions=True): except (ParseException, IndexError): return loc, ParseResults([], name=self.resultsName) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "[" + str(self.expr) + "]..." +class DelimitedList(ParseElementEnhance): + def __init__( + self, + expr: Union[str, ParserElement], + delim: Union[str, ParserElement] = ",", + combine: bool = False, + min: typing.Optional[int] = None, + max: typing.Optional[int] = None, + *, + allow_trailing_delim: bool = False, + ): + """Helper to define a delimited list of expressions - the delimiter + defaults to ','. By default, the list elements and delimiters can + have intervening whitespace, and comments, but this can be + overridden by passing ``combine=True`` in the constructor. If + ``combine`` is set to ``True``, the matching tokens are + returned as a single token string, with the delimiters included; + otherwise, the matching tokens are returned as a list of tokens, + with the delimiters suppressed. + + If ``allow_trailing_delim`` is set to True, then the list may end with + a delimiter. + + Example:: + + DelimitedList(Word(alphas)).parse_string("aa,bb,cc") # -> ['aa', 'bb', 'cc'] + DelimitedList(Word(hexnums), delim=':', combine=True).parse_string("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] + """ + if isinstance(expr, str_type): + expr = ParserElement._literalStringClass(expr) + expr = typing.cast(ParserElement, expr) + + if min is not None: + if min < 1: + raise ValueError("min must be greater than 0") + if max is not None: + if min is not None and max < min: + raise ValueError("max must be greater than, or equal to min") + + self.content = expr + self.raw_delim = str(delim) + self.delim = delim + self.combine = combine + if not combine: + self.delim = Suppress(delim) + self.min = min or 1 + self.max = max + self.allow_trailing_delim = allow_trailing_delim + + delim_list_expr = self.content + (self.delim + self.content) * ( + self.min - 1, + None if self.max is None else self.max - 1, + ) + if self.allow_trailing_delim: + delim_list_expr += Opt(self.delim) + + if self.combine: + delim_list_expr = Combine(delim_list_expr) + + super().__init__(delim_list_expr, savelist=True) + + def _generateDefaultName(self) -> str: + return "{0} [{1} {0}]...".format(self.content.streamline(), self.raw_delim) + + class _NullToken: def __bool__(self): return False @@ -4909,6 +5163,7 @@ class Opt(ParseElementEnhance): Optional matching of the given expression. Parameters: + - ``expr`` - expression that must match zero or more times - ``default`` (optional) - value to be returned if the optional expression is not found. @@ -4969,7 +5224,7 @@ def parseImpl(self, instring, loc, doActions=True): tokens = [] return loc, tokens - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: inner = str(self.expr) # strip off redundant inner {}'s while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}": @@ -4986,6 +5241,7 @@ class SkipTo(ParseElementEnhance): expression is found. Parameters: + - ``expr`` - target expression marking the end of the data to be skipped - ``include`` - if ``True``, the target expression is also parsed (the skipped text and target expression are returned as a 2-element @@ -5045,14 +5301,15 @@ def __init__( self, other: Union[ParserElement, str], include: bool = False, - ignore: bool = None, + ignore: typing.Optional[Union[ParserElement, str]] = None, fail_on: typing.Optional[Union[ParserElement, str]] = None, *, - failOn: Union[ParserElement, str] = None, + failOn: typing.Optional[Union[ParserElement, str]] = None, ): super().__init__(other) failOn = failOn or fail_on - self.ignoreExpr = ignore + if ignore is not None: + self.ignore(ignore) self.mayReturnEmpty = True self.mayIndexError = False self.includeMatch = include @@ -5070,9 +5327,7 @@ def parseImpl(self, instring, loc, doActions=True): self_failOn_canParseNext = ( self.failOn.canParseNext if self.failOn is not None else None ) - self_ignoreExpr_tryParse = ( - self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - ) + self_preParse = self.preParse if self.callPreparse else None tmploc = loc while tmploc <= instrlen: @@ -5081,13 +5336,9 @@ def parseImpl(self, instring, loc, doActions=True): if self_failOn_canParseNext(instring, tmploc): break - if self_ignoreExpr_tryParse is not None: - # advance past ignore expressions - while 1: - try: - tmploc = self_ignoreExpr_tryParse(instring, tmploc) - except ParseBaseException: - break + if self_preParse is not None: + # skip grammar-ignored expressions + tmploc = self_preParse(instring, tmploc) try: self_expr_parse(instring, tmploc, doActions=False, callPreParse=False) @@ -5145,15 +5396,20 @@ class Forward(ParseElementEnhance): def __init__(self, other: typing.Optional[Union[ParserElement, str]] = None): self.caller_frame = traceback.extract_stack(limit=2)[0] - super().__init__(other, savelist=False) + super().__init__(other, savelist=False) # type: ignore[arg-type] self.lshift_line = None - def __lshift__(self, other): + def __lshift__(self, other) -> "Forward": if hasattr(self, "caller_frame"): del self.caller_frame if isinstance(other, str_type): other = self._literalStringClass(other) + + if not isinstance(other, ParserElement): + return NotImplemented + self.expr = other + self.streamlined = other.streamlined self.mayIndexError = self.expr.mayIndexError self.mayReturnEmpty = self.expr.mayReturnEmpty self.set_whitespace_chars( @@ -5162,13 +5418,16 @@ def __lshift__(self, other): self.skipWhitespace = self.expr.skipWhitespace self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) - self.lshift_line = traceback.extract_stack(limit=2)[-2] + self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment] return self - def __ilshift__(self, other): + def __ilshift__(self, other) -> "Forward": + if not isinstance(other, ParserElement): + return NotImplemented + return self << other - def __or__(self, other): + def __or__(self, other) -> "ParserElement": caller_line = traceback.extract_stack(limit=2)[-2] if ( __diag__.warn_on_match_first_with_lshift_operator @@ -5205,12 +5464,12 @@ def parseImpl(self, instring, loc, doActions=True): not in self.suppress_warnings_ ): # walk stack until parse_string, scan_string, search_string, or transform_string is found - parse_fns = [ + parse_fns = ( "parse_string", "scan_string", "search_string", "transform_string", - ] + ) tb = traceback.extract_stack(limit=200) for i, frm in enumerate(reversed(tb), start=1): if frm.name in parse_fns: @@ -5308,6 +5567,11 @@ def streamline(self) -> ParserElement: return self def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) if validateTrace is None: validateTrace = [] @@ -5317,7 +5581,7 @@ def validate(self, validateTrace=None) -> None: self.expr.validate(tmp) self._checkRecursion([]) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: # Avoid infinite recursion by setting a temporary _defaultName self._defaultName = ": ..." @@ -5356,8 +5620,14 @@ def _setResultsName(self, name, list_all_matches=False): return super()._setResultsName(name, list_all_matches) - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + # Compatibility synonyms + # fmt: off + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... + + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + # fmt: on class TokenConverter(ParseElementEnhance): @@ -5439,11 +5709,11 @@ class Group(TokenConverter): ident = Word(alphas) num = Word(nums) term = ident | num - func = ident + Opt(delimited_list(term)) + func = ident + Opt(DelimitedList(term)) print(func.parse_string("fn a, b, 100")) # -> ['fn', 'a', 'b', '100'] - func = ident + Group(Opt(delimited_list(term))) + func = ident + Group(Opt(DelimitedList(term))) print(func.parse_string("fn a, b, 100")) # -> ['fn', ['a', 'b', '100']] """ @@ -5579,7 +5849,7 @@ class Suppress(TokenConverter): ['a', 'b', 'c', 'd'] ['START', 'relevant text ', 'END'] - (See also :class:`delimited_list`.) + (See also :class:`DelimitedList`.) """ def __init__(self, expr: Union[ParserElement, str], savelist: bool = False): @@ -5638,15 +5908,13 @@ def z(*paArgs): s, l, t = paArgs[-3:] if len(paArgs) > 3: thisFunc = paArgs[0].__class__.__name__ + "." + thisFunc - sys.stderr.write( - ">>entering {}(line: {!r}, {}, {!r})\n".format(thisFunc, line(l, s), l, t) - ) + sys.stderr.write(f">>entering {thisFunc}(line: {line(l, s)!r}, {l}, {t!r})\n") try: ret = f(*paArgs) except Exception as exc: - sys.stderr.write("< str: ) try: return "".join(_expanded(part) for part in _reBracketExpr.parse_string(s).body) - except Exception: + except Exception as e: return "" @@ -5769,7 +6037,11 @@ def autoname_elements() -> None: Utility to simplify mass-naming of parser elements, for generating railroad diagram with named subdiagrams. """ - for name, var in sys._getframe().f_back.f_locals.items(): + calling_frame = sys._getframe().f_back + if calling_frame is None: + return + calling_frame = typing.cast(types.FrameType, calling_frame) + for name, var in calling_frame.f_locals.items(): if isinstance(var, ParserElement) and not var.customName: var.set_name(name) @@ -5783,9 +6055,28 @@ def autoname_elements() -> None: ).set_name("string enclosed in single quotes") quoted_string = Combine( - Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' - | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'" -).set_name("quotedString using single or double quotes") + (Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').set_name( + "double quoted string" + ) + | (Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").set_name( + "single quoted string" + ) +).set_name("quoted string using single or double quotes") + +python_quoted_string = Combine( + (Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=re.MULTILINE) + '"""').set_name( + "multiline double quoted string" + ) + ^ ( + Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=re.MULTILINE) + "'''" + ).set_name("multiline single quoted string") + ^ (Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').set_name( + "double quoted string" + ) + ^ (Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").set_name( + "single quoted string" + ) +).set_name("Python quoted string") unicode_string = Combine("u" + quoted_string.copy()).set_name("unicode string literal") @@ -5800,9 +6091,7 @@ def autoname_elements() -> None: ] # backward compatibility names -tokenMap = token_map -conditionAsParseAction = condition_as_parse_action -nullDebugAction = null_debug_action +# fmt: off sglQuotedString = sgl_quoted_string dblQuotedString = dbl_quoted_string quotedString = quoted_string @@ -5811,4 +6100,16 @@ def autoname_elements() -> None: lineEnd = line_end stringStart = string_start stringEnd = string_end -traceParseAction = trace_parse_action + +@replaced_by_pep8(null_debug_action) +def nullDebugAction(): ... + +@replaced_by_pep8(trace_parse_action) +def traceParseAction(): ... + +@replaced_by_pep8(condition_as_parse_action) +def conditionAsParseAction(): ... + +@replaced_by_pep8(token_map) +def tokenMap(): ... +# fmt: on diff --git a/src/pip/_vendor/pyparsing/diagram/__init__.py b/src/pip/_vendor/pyparsing/diagram/__init__.py index 1506d66bf4e..83f9018ee93 100644 --- a/src/pip/_vendor/pyparsing/diagram/__init__.py +++ b/src/pip/_vendor/pyparsing/diagram/__init__.py @@ -1,3 +1,4 @@ +# mypy: ignore-errors import railroad from pip._vendor import pyparsing import typing @@ -17,11 +18,13 @@ jinja2_template_source = """\ +{% if not embed %} +{% endif %} {% if not head %} -