diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3d9e4e6b..50090cfe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,42 @@ Changelog ========= +v0.13.2 +----------- + +- Speed up downloads with asyncio + +- New settings featuring environment variables and .env file to store settings and defaults. + + - This also changes the CACHE_THIRDPARTY_DIR environment variable: it used to default first + to ".cache/python_inspector" and if not writable, it would fallback to home + "~/.cache/python_inspector". The new behavior is to only use the "~/.cache/python_inspector" + in the home directory. You can configure this directory to any other value. + + - Another change is that pypi.org is no longer systematically added as an index URL for + resolution. Instead the list of configured index URLs is used, and only defaults to pypi.org + if not provided. + + - Another change is that it is possible to only use the provided or configured index URLs + and skip other URLs from requirements that are not in these configured URLs. + + - Calling utils_pypi.download_sdist or utils_pypi.download_wheel requires a non-empty list + of PypiSimpleRepository. + + - python_inspector.utils_pypi.Distribution.download_url is now a method, not a property + + - The command line has again a default OS and Python version set. + + - Default option values are reported in the JSON results. They were skipped before. + +- Drop support for running on Python 3.8. You can still resolve dependencies for Python 3.8. + The default command line tool Python version used for resolution is now 3.9. + +- Add support for the latest Python and OS versions. + +- Merge latest skeleton and adopt ruff for code formatting. + + v0.13.1 ----------- diff --git a/requirements.txt b/requirements.txt index f64e8884..8983c3ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,8 @@ packaging==24.2 packvers==21.5 pip-requirements-parser==32.0.1 pkginfo2==30.0.0 +pydantic_settings == 2.8.1 +pydantic == 2.11.2 pyparsing==3.0.9 PyYAML==6.0 requests==2.28.1 diff --git a/setup.cfg b/setup.cfg index 7ec4f263..a4eead5c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -71,6 +71,8 @@ install_requires = packvers >= 21.5 aiohttp >= 3.8 aiofiles >= 23.1 + pydantic >= 2.10.0 + pydantic_settings >= 2.8.0 [options.packages.find] where = src diff --git a/src/python_inspector/__init__.py b/src/python_inspector/__init__.py index a990b071..8d70e3d6 100644 --- a/src/python_inspector/__init__.py +++ b/src/python_inspector/__init__.py @@ -7,4 +7,9 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -DEFAULT_PYTHON_VERSION = "3.8" +from python_inspector import settings + +# Initialize global settings +pyinspector_settings = settings.Settings() + +settings.create_cache_directory(pyinspector_settings.CACHE_THIRDPARTY_DIR) diff --git a/src/python_inspector/api.py b/src/python_inspector/api.py index 2a373227..f820fae3 100644 --- a/src/python_inspector/api.py +++ b/src/python_inspector/api.py @@ -29,6 +29,7 @@ from _packagedcode.pypi import can_process_dependent_package from _packagedcode.pypi import get_resolved_purl from python_inspector import dependencies +from python_inspector import pyinspector_settings as settings from python_inspector import utils from python_inspector import utils_pypi from python_inspector.package_data import get_pypi_data_from_purl @@ -42,8 +43,8 @@ from python_inspector.resolution import get_requirements_from_python_manifest from python_inspector.utils import Candidate from python_inspector.utils_pypi import PLATFORMS_BY_OS -from python_inspector.utils_pypi import PYPI_SIMPLE_URL from python_inspector.utils_pypi import Environment +from python_inspector.utils_pypi import PypiSimpleRepository from python_inspector.utils_pypi import valid_python_versions @@ -80,7 +81,7 @@ def resolve_dependencies( specifiers=tuple(), python_version=None, operating_system=None, - index_urls=tuple([PYPI_SIMPLE_URL]), + index_urls: tuple[str, ...] = settings.INDEX_URL, pdt_output=None, netrc_file=None, max_rounds=200000, @@ -103,7 +104,7 @@ def resolve_dependencies( linux OS. Download from the provided PyPI simple index_urls INDEX(s) URLs defaulting - to PyPI.org + to PyPI.org or a configured setting. """ if not operating_system: @@ -148,9 +149,6 @@ def resolve_dependencies( files = [] - if PYPI_SIMPLE_URL not in index_urls: - index_urls = tuple([PYPI_SIMPLE_URL]) + tuple(index_urls) - # requirements for req_file in requirement_files: deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file) @@ -249,29 +247,32 @@ def resolve_dependencies( if verbose: printer(f"environment: {environment}") - repos = [] + repos_by_url = {} if not use_pypi_json_api: # Collect PyPI repos + use_only_confed = settings.USE_ONLY_CONFIGURED_INDEX_URLS for index_url in index_urls: index_url = index_url.strip("/") - existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) - if existing: - existing.use_cached_index = use_cached_index - repos.append(existing) - else: - credentials = None - if parsed_netrc: - login, password = utils.get_netrc_auth(index_url, parsed_netrc) - credentials = ( - dict(login=login, password=password) if login and password else None - ) - repo = utils_pypi.PypiSimpleRepository( - index_url=index_url, - use_cached_index=use_cached_index, - credentials=credentials, - ) - repos.append(repo) + if use_only_confed and index_url not in settings.INDEX_URL: + if verbose: + printer(f"Skipping index URL unknown in settings: {index_url!r}") + continue + if index_url in repos_by_url: + continue + + credentials = None + if parsed_netrc: + login, password = utils.get_netrc_auth(index_url, parsed_netrc) + if login and password: + credentials = dict(login=login, password=password) + repo = utils_pypi.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + credentials=credentials, + ) + repos_by_url[index_url] = repo + repos = repos_by_url.values() if verbose: printer("repos:") for repo in repos: diff --git a/src/python_inspector/package_data.py b/src/python_inspector/package_data.py index 86aa0231..6b0a9733 100644 --- a/src/python_inspector/package_data.py +++ b/src/python_inspector/package_data.py @@ -168,7 +168,7 @@ async def get_wheel_download_urls( environment=environment, python_version=python_version, ): - download_urls.append(await wheel.download_url) + download_urls.append(await wheel.download_url(repo)) return download_urls @@ -186,4 +186,4 @@ async def get_sdist_download_url( python_version=python_version, ) if sdist: - return await sdist.download_url + return await sdist.download_url(repo) diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index da516d61..6dca3a5d 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -39,6 +39,7 @@ from _packagedcode.pypi import PythonSetupPyHandler from _packagedcode.pypi import SetupCfgHandler from _packagedcode.pypi import can_process_dependent_package +from python_inspector import pyinspector_settings as settings from python_inspector import utils_pypi from python_inspector.error import NoVersionsFound from python_inspector.setup_py_live_eval import iter_requirements @@ -107,7 +108,7 @@ def get_deps_from_distribution( deps = [] for package_data in handler.parse(location): dependencies = package_data.dependencies - deps.extend(dependencies) + deps.extend(dependencies=dependencies) return deps @@ -211,21 +212,21 @@ async def fetch_and_extract_sdist( def get_sdist_file_path_from_filename(sdist): if sdist.endswith(".tar.gz"): sdist_file = sdist.rstrip(".tar.gz") - with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file: + with tarfile.open(os.path.join(settings.CACHE_THIRDPARTY_DIR, sdist)) as file: file.extractall( - os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file) + os.path.join(settings.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file) ) elif sdist.endswith(".zip"): sdist_file = sdist.rstrip(".zip") - with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as zip: + with ZipFile(os.path.join(settings.CACHE_THIRDPARTY_DIR, sdist)) as zip: zip.extractall( - os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file) + os.path.join(settings.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file) ) else: raise Exception(f"Unable to extract sdist {sdist}") - return os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file) + return os.path.join(settings.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file) def get_requirements_from_dependencies( @@ -444,7 +445,7 @@ async def _get_versions_for_package_from_pypi_json_api(self, name: str) -> List[ api_url = f"https://pypi.org/pypi/{name}/json" resp = await get_response_async(api_url) if not resp: - self.versions_by_package[name] = [] + return [] releases = resp.get("releases") or {} return releases.keys() or [] @@ -497,7 +498,7 @@ async def _get_requirements_for_package_from_pypi_simple( if wheels: for wheel in wheels: - wheel_location = os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel) + wheel_location = os.path.join(settings.CACHE_THIRDPARTY_DIR, wheel) requirements = get_requirements_from_distribution( handler=PypiWheelHandler, location=wheel_location, @@ -596,7 +597,8 @@ def _iter_matches( name = remove_extras(identifier=identifier) bad_versions = {c.version for c in incompatibilities[identifier]} extras = {e for r in requirements[identifier] for e in r.extras} - versions = self.get_versions_for_package(name) + versions = [] + versions.extend(self.get_versions_for_package(name=name)) if not versions: if self.ignore_errors: diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index b02710b1..0200eb40 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -13,6 +13,7 @@ import click +from python_inspector import pyinspector_settings as settings from python_inspector import utils_pypi from python_inspector.cli_utils import FileOptionType from python_inspector.utils import write_output_in_file @@ -21,8 +22,7 @@ __version__ = "0.13.0" -DEFAULT_PYTHON_VERSION = "38" -PYPI_SIMPLE_URL = "https://pypi.org/simple" +DEFAULT_PYTHON_VERSION = settings.DEFAULT_PYTHON_VERSION def print_version(ctx, param, value): @@ -71,6 +71,7 @@ def print_version(ctx, param, value): "python_version", type=click.Choice(utils_pypi.valid_python_versions), metavar="PYVER", + default=settings.DEFAULT_PYTHON_VERSION, show_default=True, required=True, help="Python version to use for dependency resolution. One of " @@ -82,6 +83,7 @@ def print_version(ctx, param, value): "operating_system", type=click.Choice(utils_pypi.PLATFORMS_BY_OS), metavar="OS", + default=settings.DEFAULT_OS, show_default=True, required=True, help="OS to use for dependency resolution. One of " + ", ".join(utils_pypi.PLATFORMS_BY_OS), @@ -92,7 +94,7 @@ def print_version(ctx, param, value): type=str, metavar="INDEX", show_default=True, - default=tuple([PYPI_SIMPLE_URL]), + default=tuple(settings.INDEX_URL), multiple=True, help="PyPI simple index URL(s) to use in order of preference. " "This option can be used multiple times.", @@ -319,9 +321,6 @@ def get_pretty_options(ctx, generic_paths=False): if getattr(param, "hidden", False): continue - if value == param.default: - continue - if value in (None, False): continue diff --git a/src/python_inspector/settings.py b/src/python_inspector/settings.py new file mode 100644 index 00000000..5786bb11 --- /dev/null +++ b/src/python_inspector/settings.py @@ -0,0 +1,61 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path + +from pydantic import field_validator +from pydantic_settings import BaseSettings +from pydantic_settings import SettingsConfigDict + + +class Settings(BaseSettings): + """ + Reference: https://docs.pydantic.dev/latest/concepts/pydantic_settings/ + A settings object: use it with an .env file and/or environment variables all prefixed with + PYTHON_INSPECTOR_ + """ + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + env_prefix="PYTHON_INSPECTOR_", + case_sensitive=True, + extra="allow", + ) + + # the default Python version to use if none is provided + DEFAULT_PYTHON_VERSION: str = "39" + + # the default OS to use if none is provided + DEFAULT_OS: str = "linux" + + # a list of PyPI simple index URLs. Use a JSON array to represent multiple URLs + INDEX_URL: tuple[str, ...] = ("https://pypi.org/simple",) + + # If True, only uses configured INDEX_URLs listed above and ignore other URLs found in requirements + USE_ONLY_CONFIGURED_INDEX_URLS: bool = False + + # a path string where to store the cached downloads. Will be created if it does not exists. + CACHE_THIRDPARTY_DIR: str = str(Path(Path.home() / ".cache/python_inspector")) + + @field_validator("INDEX_URL") + @classmethod + def validate_index_url(cls, value): + if isinstance(value, str): + return (value,) + elif isinstance(value, (tuple, list)): + return tuple(value) + else: + raise ValueError(f"INDEX_URL must be either a URL or list of URLs: {value!r}") + + +def create_cache_directory(cache_dir): + cache_dir = Path(cache_dir).expanduser().resolve().absolute() + if not cache_dir.exists(): + cache_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index 481a2a1e..058fd15c 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -41,7 +41,7 @@ from packvers import version as packaging_version from packvers.specifiers import SpecifierSet -from python_inspector import DEFAULT_PYTHON_VERSION +from python_inspector import pyinspector_settings as settings from python_inspector import utils_pip_compatibility_tags """ @@ -192,21 +192,9 @@ def get_python_dot_version(version): ], } -CACHE_THIRDPARTY_DIR = os.environ.get("PYTHON_INSPECTOR_CACHE_DIR") -if not CACHE_THIRDPARTY_DIR: - CACHE_THIRDPARTY_DIR = ".cache/python_inspector" - try: - os.makedirs(CACHE_THIRDPARTY_DIR, exist_ok=True) - except Exception: - home = pathlib.Path.home() - CACHE_THIRDPARTY_DIR = str(home / ".cache/python_inspector") - os.makedirs(CACHE_THIRDPARTY_DIR, exist_ok=True) - +DEFAULT_PYTHON_VERSION = settings.DEFAULT_PYTHON_VERSION +CACHE_THIRDPARTY_DIR = settings.CACHE_THIRDPARTY_DIR -################################################################################ - -PYPI_SIMPLE_URL = "https://pypi.org/simple" -PYPI_INDEX_URLS = (PYPI_SIMPLE_URL,) ################################################################################ @@ -250,7 +238,7 @@ async def download_wheel( print(f" download_wheel: {name}=={version} for envt: {environment}") if not repos: - repos = DEFAULT_PYPI_REPOS + raise ValueError("download_wheel: missing repos") fetched_wheel_filenames = [] for repo in repos: @@ -328,7 +316,7 @@ async def get_supported_and_valid_wheels( if TRACE_DEEP: print( f""" get_supported_and_valid_wheels: Getting wheel from index (or cache): - {wheel.download_url}""" + {await wheel.download_url(repo)}""" ) wheels.append(wheel) return wheels @@ -363,7 +351,7 @@ async def download_sdist( print(f" download_sdist: {name}=={version}") if not repos: - repos = DEFAULT_PYPI_REPOS + raise ValueError("download_sdist: missing repos") fetched_sdist_filename = None @@ -618,9 +606,10 @@ def package_url(self): ) ) - @property - async def download_url(self): - return await self.get_best_download_url() + async def download_url(self, repo): + if not repo: + raise ValueError("download_url: missing repo") + return await self.get_best_download_url(repos=(repo,)) async def get_best_download_url(self, repos=tuple()): """ @@ -632,7 +621,7 @@ async def get_best_download_url(self, repos=tuple()): """ if not repos: - repos = DEFAULT_PYPI_REPOS + raise ValueError("get_best_download_url: missing repos") for repo in repos: package = await repo.get_package_version(name=self.name, version=self.version) @@ -1340,9 +1329,6 @@ async def dists_from_links(cls, links: List[Link]): " ===> dists_from_paths_or_urls:", dist, "\n ", - "with URL:", - await dist.download_url, - "\n ", "from URL:", link.url, ) @@ -1476,7 +1462,8 @@ class PypiSimpleRepository: index_url = attr.ib( type=str, - default=PYPI_SIMPLE_URL, + # We use first entry in INDEX_URL setting as default + default=settings.INDEX_URL[0], metadata=dict(help="Base PyPI simple URL for this index."), ) @@ -1656,10 +1643,6 @@ def resolve_relative_url(package_url, url): return url -PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) -DEFAULT_PYPI_REPOS = (PYPI_PUBLIC_REPO,) -DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} - ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1859,3 +1842,10 @@ async def fetch_and_save( os.symlink(os.path.abspath(path), output) return content + + +def get_current_indexes() -> list[PypiSimpleRepository]: + """ + Return a list of PypiSimpleRepository indexes configured in settings. + """ + return [PypiSimpleRepository(index_url=url) for url in settings.INDEX_URL] diff --git a/tests/data/azure-devops.req-310-expected.json b/tests/data/azure-devops.req-310-expected.json index a4220915..157b982a 100644 --- a/tests/data/azure-devops.req-310-expected.json +++ b/tests/data/azure-devops.req-310-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 310", diff --git a/tests/data/azure-devops.req-312-expected.json b/tests/data/azure-devops.req-312-expected.json index fa29754b..90640eca 100644 --- a/tests/data/azure-devops.req-312-expected.json +++ b/tests/data/azure-devops.req-312-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 312", diff --git a/tests/data/azure-devops.req-313-expected.json b/tests/data/azure-devops.req-313-expected.json index a03a1e01..6d71dbc0 100644 --- a/tests/data/azure-devops.req-313-expected.json +++ b/tests/data/azure-devops.req-313-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 313", diff --git a/tests/data/azure-devops.req-38-expected.json b/tests/data/azure-devops.req-38-expected.json index 30f38d74..a3b820a8 100644 --- a/tests/data/azure-devops.req-38-expected.json +++ b/tests/data/azure-devops.req-38-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/default-url-expected.json b/tests/data/default-url-expected.json index ea81f023..4b6ea4b2 100644 --- a/tests/data/default-url-expected.json +++ b/tests/data/default-url-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/environment-marker-test-requirements.txt-expected.json b/tests/data/environment-marker-test-requirements.txt-expected.json index 87144f2e..78ebcee9 100644 --- a/tests/data/environment-marker-test-requirements.txt-expected.json +++ b/tests/data/environment-marker-test-requirements.txt-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json-pdt ", "--operating-system linux", "--python-version 37", diff --git a/tests/data/example-requirements-ignore-errors-expected.json b/tests/data/example-requirements-ignore-errors-expected.json index 623ac28b..9a797f66 100644 --- a/tests/data/example-requirements-ignore-errors-expected.json +++ b/tests/data/example-requirements-ignore-errors-expected.json @@ -5,6 +5,7 @@ "tool_version": "0.13.0", "options": [ "--ignore-errors", + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 310", diff --git a/tests/data/frozen-requirements.txt-expected.json b/tests/data/frozen-requirements.txt-expected.json index b34c8340..dea7210c 100644 --- a/tests/data/frozen-requirements.txt-expected.json +++ b/tests/data/frozen-requirements.txt-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json-pdt ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/insecure-setup-2/setup.py-expected.json b/tests/data/insecure-setup-2/setup.py-expected.json index f300166e..86c627ec 100644 --- a/tests/data/insecure-setup-2/setup.py-expected.json +++ b/tests/data/insecure-setup-2/setup.py-expected.json @@ -5,6 +5,7 @@ "tool_version": "0.13.0", "options": [ "--analyze-setup-py-insecurely", + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 27", diff --git a/tests/data/insecure-setup/setup.py-expected.json b/tests/data/insecure-setup/setup.py-expected.json index 8c1587d9..65279116 100644 --- a/tests/data/insecure-setup/setup.py-expected.json +++ b/tests/data/insecure-setup/setup.py-expected.json @@ -5,6 +5,7 @@ "tool_version": "0.13.0", "options": [ "--analyze-setup-py-insecurely", + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 27", diff --git a/tests/data/no-install-requires-expected.json b/tests/data/no-install-requires-expected.json index ab0080db..54f03fde 100644 --- a/tests/data/no-install-requires-expected.json +++ b/tests/data/no-install-requires-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/pdt-requirements.txt-expected.json b/tests/data/pdt-requirements.txt-expected.json index 47c60f33..5611bdcf 100644 --- a/tests/data/pdt-requirements.txt-expected.json +++ b/tests/data/pdt-requirements.txt-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json-pdt ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/pinned-pdt-requirements.txt-expected.json b/tests/data/pinned-pdt-requirements.txt-expected.json index e66ccda7..3ea67301 100644 --- a/tests/data/pinned-pdt-requirements.txt-expected.json +++ b/tests/data/pinned-pdt-requirements.txt-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json-pdt ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/pinned-requirements.txt-expected.json b/tests/data/pinned-requirements.txt-expected.json index b767ec59..0cdd7c90 100644 --- a/tests/data/pinned-requirements.txt-expected.json +++ b/tests/data/pinned-requirements.txt-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/prefer-source-expected.json b/tests/data/prefer-source-expected.json index 578b880c..86d019b5 100644 --- a/tests/data/prefer-source-expected.json +++ b/tests/data/prefer-source-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--prefer-source", diff --git a/tests/data/setup/no-direct-dependencies-setup.py-expected.json b/tests/data/setup/no-direct-dependencies-setup.py-expected.json index 206c5d42..c9664ad7 100644 --- a/tests/data/setup/no-direct-dependencies-setup.py-expected.json +++ b/tests/data/setup/no-direct-dependencies-setup.py-expected.json @@ -5,6 +5,7 @@ "tool_version": "0.13.0", "options": [ "--analyze-setup-py-insecurely", + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 27", diff --git a/tests/data/setup/simple-setup.py-expected.json b/tests/data/setup/simple-setup.py-expected.json index cc9787d5..5a5e3a23 100644 --- a/tests/data/setup/simple-setup.py-expected.json +++ b/tests/data/setup/simple-setup.py-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 27", diff --git a/tests/data/setup/spdx-setup.py-expected.json b/tests/data/setup/spdx-setup.py-expected.json index 0a4be9d5..60c2dfff 100644 --- a/tests/data/setup/spdx-setup.py-expected.json +++ b/tests/data/setup/spdx-setup.py-expected.json @@ -5,6 +5,7 @@ "tool_version": "0.13.0", "options": [ "--analyze-setup-py-insecurely", + "--index-url https://pypi.org/simple", "--json-pdt ", "--operating-system linux", "--python-version 27", diff --git a/tests/data/single-url-except-simple-expected.json b/tests/data/single-url-except-simple-expected.json index 6ddb482c..af6ee25a 100644 --- a/tests/data/single-url-except-simple-expected.json +++ b/tests/data/single-url-except-simple-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple/", "--index-url https://thirdparty.aboutcode.org/pypi/simple/", "--json ", "--operating-system linux", @@ -353,63 +354,6 @@ "datasource_id": null, "purl": "pkg:pypi/jinja2@3.1.6" }, - { - "type": "pypi", - "namespace": null, - "name": "markupsafe", - "version": "2.1.5", - "qualifiers": {}, - "subpath": null, - "primary_language": "Python", - "description": "Safely add untrusted strings to HTML/XML markup.\nMarkupSafe\n==========\n\nMarkupSafe implements a text object that escapes characters so it is\nsafe to use in HTML and XML. Characters that have special meanings are\nreplaced so that they display as the actual characters. This mitigates\ninjection attacks, meaning untrusted user input can safely be displayed\non a page.\n\n\nInstalling\n----------\n\nInstall and update using `pip`_:\n\n.. code-block:: text\n\n pip install -U MarkupSafe\n\n.. _pip: https://pip.pypa.io/en/stable/getting-started/\n\n\nExamples\n--------\n\n.. code-block:: pycon\n\n >>> from markupsafe import Markup, escape\n\n >>> # escape replaces special characters and wraps in Markup\n >>> escape(\"\")\n Markup('<script>alert(document.cookie);</script>')\n\n >>> # wrap in Markup to mark text \"safe\" and prevent escaping\n >>> Markup(\"Hello\")\n Markup('hello')\n\n >>> escape(Markup(\"Hello\"))\n Markup('hello')\n\n >>> # Markup is a str subclass\n >>> # methods and operators escape their arguments\n >>> template = Markup(\"Hello {name}\")\n >>> template.format(name='\"World\"')\n Markup('Hello "World"')\n\n\nDonate\n------\n\nThe Pallets organization develops and supports MarkupSafe and other\npopular packages. In order to grow the community of contributors and\nusers, and allow the maintainers to devote more time to the projects,\n`please donate today`_.\n\n.. _please donate today: https://palletsprojects.com/donate\n\n\nLinks\n-----\n\n- Documentation: https://markupsafe.palletsprojects.com/\n- Changes: https://markupsafe.palletsprojects.com/changes/\n- PyPI Releases: https://pypi.org/project/MarkupSafe/\n- Source Code: https://github.com/pallets/markupsafe/\n- Issue Tracker: https://github.com/pallets/markupsafe/issues/\n- Chat: https://discord.gg/pallets", - "release_date": "2024-02-02T16:31:01", - "parties": [ - { - "type": "person", - "role": "maintainer", - "name": "Pallets", - "email": "contact@palletsprojects.com", - "url": null - } - ], - "keywords": [ - "Development Status :: 5 - Production/Stable", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Topic :: Internet :: WWW/HTTP :: Dynamic Content", - "Topic :: Text Processing :: Markup :: HTML" - ], - "homepage_url": "https://palletsprojects.com/p/markupsafe/", - "download_url": "https://files.pythonhosted.org/packages/c7/bd/50319665ce81bb10e90d1cf76f9e1aa269ea6f7fa30ab4521f14d122a3df/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "size": 26106, - "sha1": null, - "md5": "4f97754a1154496e5bc9d3f21fb0315a", - "sha256": "fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", - "sha512": null, - "bug_tracking_url": "https://github.com/pallets/markupsafe/issues/", - "code_view_url": "https://github.com/pallets/markupsafe/", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": { - "license": "BSD-3-Clause", - "classifiers": [ - "License :: OSI Approved :: BSD License" - ] - }, - "notice_text": null, - "source_packages": [], - "file_references": [], - "extra_data": {}, - "dependencies": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": "https://pypi.org/pypi/markupsafe/2.1.5/json", - "datasource_id": null, - "purl": "pkg:pypi/markupsafe@2.1.5" - }, { "type": "pypi", "namespace": null, diff --git a/tests/data/single-url-expected.json b/tests/data/single-url-expected.json index 082fdb76..48db55af 100644 --- a/tests/data/single-url-expected.json +++ b/tests/data/single-url-expected.json @@ -4,6 +4,7 @@ "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector", "tool_version": "0.13.0", "options": [ + "--index-url https://pypi.org/simple", "--json ", "--operating-system linux", "--python-version 38", diff --git a/tests/data/tilde_req-expected.json b/tests/data/tilde_req-expected.json index 85d21c53..de2635f7 100644 --- a/tests/data/tilde_req-expected.json +++ b/tests/data/tilde_req-expected.json @@ -26,7 +26,7 @@ "subpath": null, "primary_language": "Python", "description": "Backport of pathlib-compatible object wrapper for zip files\n.. image:: https://img.shields.io/pypi/v/zipp.svg\n :target: `PyPI link`_\n\n.. image:: https://img.shields.io/pypi/pyversions/zipp.svg\n :target: `PyPI link`_\n\n.. _PyPI link: https://pypi.org/project/zipp\n\n.. image:: https://github.com/jaraco/zipp/workflows/tests/badge.svg\n :target: https://github.com/jaraco/zipp/actions?query=workflow%3A%22tests%22\n :alt: tests\n\n.. image:: https://img.shields.io/badge/code%20style-black-000000.svg\n :target: https://github.com/psf/black\n :alt: Code style: Black\n\n.. .. image:: https://readthedocs.org/projects/skeleton/badge/?version=latest\n.. :target: https://skeleton.readthedocs.io/en/latest/?badge=latest\n\n.. image:: https://img.shields.io/badge/skeleton-2022-informational\n :target: https://blog.jaraco.com/skeleton\n\n.. image:: https://tidelift.com/badges/package/pypi/zipp\n :target: https://tidelift.com/subscription/pkg/pypi-zipp?utm_source=pypi-zipp&utm_medium=readme\n\n\nA pathlib-compatible Zipfile object wrapper. Official backport of the standard library\n`Path object `_.\n\n\nCompatibility\n=============\n\nNew features are introduced in this third-party library and later merged\ninto CPython. The following table indicates which versions of this library\nwere contributed to different versions in the standard library:\n\n.. list-table::\n :header-rows: 1\n\n * - zipp\n - stdlib\n * - 3.5\n - 3.11\n * - 3.3\n - 3.9\n * - 1.0\n - 3.8\n\n\nUsage\n=====\n\nUse ``zipp.Path`` in place of ``zipfile.Path`` on any Python.\n\nFor Enterprise\n==============\n\nAvailable as part of the Tidelift Subscription.\n\nThis project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.\n\n`Learn more `_.\n\nSecurity Contact\n================\n\nTo report a security vulnerability, please use the\n`Tidelift security contact `_.\nTidelift will coordinate the fix and disclosure.", - "release_date": "2022-07-12T14:21:20", + "release_date": "2022-07-12T14:21:21", "parties": [ { "type": "person", @@ -43,11 +43,11 @@ "Programming Language :: Python :: 3 :: Only" ], "homepage_url": "https://github.com/jaraco/zipp", - "download_url": "https://files.pythonhosted.org/packages/f0/36/639d6742bcc3ffdce8b85c31d79fcfae7bb04b95f0e5c4c6f8b206a038cc/zipp-3.8.1-py3-none-any.whl", - "size": 5645, + "download_url": "https://files.pythonhosted.org/packages/3b/e3/fb79a1ea5f3a7e9745f688855d3c673f2ef7921639a380ec76f7d4d83a85/zipp-3.8.1.tar.gz", + "size": 14189, "sha1": null, - "md5": "300aa262796e7ebfb57b4d6731821c29", - "sha256": "47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009", + "md5": "6f15c3e3c78919f8936749b0033e0cea", + "sha256": "05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2", "sha512": null, "bug_tracking_url": null, "code_view_url": null, diff --git a/tests/test_cli.py b/tests/test_cli.py index a9abb303..f4b70f99 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -145,6 +145,8 @@ def test_cli_with_single_index_url_except_pypi_simple(): extra_options = [ "--index-url", "https://thirdparty.aboutcode.org/pypi/simple/", + "--index-url", + "https://pypi.org/simple/", ] check_specs_resolution( specifier=specifier, @@ -468,12 +470,30 @@ def test_passing_of_no_json_output_flag(): def test_passing_of_no_os(): - options = ["--specifier", "foo", "--json", "-", "--python-version", "38"] + options = [ + "--specifier", + "foo", + "--json", + "-", + "--python-version", + "38", + "--operating-system", + "", + ] run_cli(options=options, expected_rc=2, get_env=False) def test_passing_of_no_pyver(): - options = ["--specifier", "foo", "--json", "-", "--operating-system", "linux"] + options = [ + "--specifier", + "foo", + "--json", + "-", + "--operating-system", + "linux", + "--python-version", + "", + ] run_cli(options=options, expected_rc=2, get_env=False) diff --git a/tests/test_resolution.py b/tests/test_resolution.py index 9ad953d3..219f02c1 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -26,9 +26,9 @@ from python_inspector.resolution import get_requirements_from_python_manifest from python_inspector.resolution import is_valid_version from python_inspector.resolution import parse_reqs_from_setup_py_insecurely -from python_inspector.utils_pypi import PYPI_PUBLIC_REPO from python_inspector.utils_pypi import Environment from python_inspector.utils_pypi import PypiSimpleRepository +from python_inspector.utils_pypi import get_current_indexes # Used for tests to regenerate fixtures with regen=True REGEN_TEST_FIXTURES = os.getenv("PYINSP_REGEN_TEST_FIXTURES", False) @@ -52,7 +52,7 @@ def check_get_resolved_dependencies( get_resolved_dependencies( requirements=[requirement], environment=env, - repos=repos or [PYPI_PUBLIC_REPO], + repos=repos or get_current_indexes(), as_tree=as_tree, ) ) @@ -319,7 +319,8 @@ def test_setup_py_parsing_insecure_testpkh(): @patch("python_inspector.resolution.PythonInputProvider.get_versions_for_package") def test_iter_matches(mock_versions): + repos = get_current_indexes() mock_versions.return_value = [] - provider = PythonInputProvider() + provider = PythonInputProvider(repos=repos) with pytest.raises(NoVersionsFound): list(provider._iter_matches("foo-bar", {"foo-bar": []}, {"foo-bar": []}))