From d558699c08821cbcdc3dac87fdc7594e3846ba5f Mon Sep 17 00:00:00 2001 From: Faustin Carter Date: Fri, 24 May 2024 12:23:49 -0700 Subject: [PATCH 1/3] Add in new --pypi-mirror-url and --pypi-metadata-url options and deprecate --pypi-url --- README.md | 21 ++++++++++++++++++++ grayskull/config.py | 11 +++++++---- grayskull/main.py | 40 +++++++++++++++++++++++++++++++++----- grayskull/strategy/pypi.py | 8 ++++---- tests/cli/test_cli_cmds.py | 1 + 5 files changed, 68 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 16122e991..bdd0f2367 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,27 @@ grayskull pypi ./pytest-5.3.5.tar.gz Note that such a recipe isn't really portable as it will depend on the local path of the sdist file. It can be useful if you want to automatically generate a conda package. +### Use Grayskull with an internal package index + +Grayskull can create recipes that point to any Python Package Index. Supply the `--pypi-mirror-url` keyword. + +* Example: +```bash +grayskull pypi --pypi-mirror-url https://pypi.example.com pytest +``` + +The above will source packages from `https://pypi.example.com/packages/source/...` + +This assumes that the mirror follows the same API as pypi _including_ hosting metadata at the `/pypi/{package_name}/json` endpoint. +To specify an alternate metadata location use the `--pypi-metadata-url` option. + +* Example: +```bash +grayskull pypi --pypi-mirror-url https://pypi.example.com --pypi-metadata-url https://pypi_meta.example.com pytest +``` + +> *Note:* `--pypi-metadata-url` is a replacement for `--pypi-url`; `--pypi-url` is deprecated and will be removed in a future release. + ### Online Grayskull It is also possible to use Grayskull without any installation. You can go to this website [marcelotrevisani.com/grayskull](https://www.marcelotrevisani.com/grayskull) and inform the name and the version (optional) of the package and it will create the recipe for you. diff --git a/grayskull/config.py b/grayskull/config.py index 544095028..d0068a03f 100644 --- a/grayskull/config.py +++ b/grayskull/config.py @@ -4,6 +4,9 @@ from grayskull.cli.parser import parse_pkg_name_version from grayskull.utils import PyVer +DEFAULT_PYPI_URL = "https://pypi.org" +DEFAULT_PYPI_META_URL = "https://pypi.org/pypi" + @dataclass class Configuration: @@ -37,7 +40,8 @@ class Configuration: default_factory=lambda: ("cython", "cython-blis", "blis") ) pkg_need_cxx_compiler: Tuple = field(default_factory=lambda: ("pybind11",)) - url_pypi_metadata: str = "https://pypi.org/pypi/{pkg_name}/json" + url_pypi: str = DEFAULT_PYPI_URL + url_pypi_metadata: str = DEFAULT_PYPI_META_URL download: bool = False is_arch: bool = False repo_github: Optional[str] = None @@ -103,11 +107,10 @@ def get_py_version_available( return py_ver_enabled def __post_init__(self): + if not self.url_pypi_metadata.endswith("/{pkg_name}/json"): + self.url_pypi_metadata = self.url_pypi_metadata.rstrip("/") + "/{pkg_name}/json" if self.from_local_sdist: self.local_sdist = self.local_sdist or self.name - if self.url_pypi_metadata != "https://pypi.org/pypi/{pkg_name}/json": - prefix = "" if self.url_pypi_metadata.endswith("/") else "/" - self.url_pypi_metadata += f"{prefix}{{pkg_name}}/json" pkg_repo, pkg_name, pkg_version = parse_pkg_name_version(self.name) if pkg_repo: prefix = "" if pkg_repo.endswith("/") else "/" diff --git a/grayskull/main.py b/grayskull/main.py index 4c385e530..1cf9c31e3 100644 --- a/grayskull/main.py +++ b/grayskull/main.py @@ -13,7 +13,7 @@ from grayskull.base.github import get_git_current_user from grayskull.cli import CLIConfig from grayskull.cli.stdout import print_msg -from grayskull.config import Configuration +from grayskull.config import DEFAULT_PYPI_META_URL, DEFAULT_PYPI_URL, Configuration from grayskull.utils import generate_recipe, origin_is_github, origin_is_local_sdist init(autoreset=True) @@ -161,10 +161,23 @@ def init_parser(): help="It will generate the recipes strict for the conda-forge channel.", ) pypi_parser.add_argument( - "--pypi-url", - default="https://pypi.org/pypi/", + "--pypi-metadata-url", + default=DEFAULT_PYPI_META_URL, dest="url_pypi_metadata", - help="Pypi url server", + help="Pypi url server metadata endpoint; will be appended with '{pkgname}/json'", + ) + pypi_parser.add_argument( + "--pypi-mirror-url", + default=DEFAULT_PYPI_URL, + dest="url_pypi_mirror", + help="Pypi mirror URL; assumed to have same API as pypi.org", + ) + # TODO: Remove before 3.0 release + pypi_parser.add_argument( + "--pypi-url", + default=None, + dest="url_pypi_metadata_deprecated", + help="DEPRECATED: use --pypi-metadata-url instead", ) pypi_parser.add_argument( "--recursive", @@ -297,11 +310,28 @@ def generate_recipes_from_list(list_pkgs, args): if Path(pkg_name).is_file() and (not from_local_sdist): args.output = pkg_name try: + # TODO: Remove before 3.0 release + if args.url_pypi_metadata_deprecated and args.url_pypi_metadata: + raise RuntimeError("--pypi-url is deprecated in favor of --pypi-url-metadata and may not be passed in conjunction with --pypi-url-metadata") + + # TODO: Remove before 3.0 release + if args.url_pypi_metadata_deprecated is not None: + logging.warning("--pypi-url is deprecated; use --pypi-url-metadata instead") + args.url_pypi_metadata = args.url_pypi_metadata_deprecated + + # If a PYPI mirror is selected, but the metadata URL is not + # explicitly passed, assume the mirror can handle the standard + # metadata endpoint and coerce the metadata URL appropriately in a + # way that respects the DEFAULT settings from config. + if (args.url_pypi_mirror.rstrip("/") != DEFAULT_PYPI_URL) and (args.url_pypi_metadata.rstrip("/") == DEFAULT_PYPI_META_URL): + args.url_pypi_metadata = DEFAULT_PYPI_META_URL.replace(DEFAULT_PYPI_URL, args.url_pypi_mirror.rstrip("/")) + recipe, config = create_python_recipe( pkg_name, is_strict_cf=args.is_strict_conda_forge, download=args.download, - url_pypi_metadata=args.url_pypi_metadata, + url_pypi=args.url_pypi_mirror.rstrip("/"), + url_pypi_metadata=args.url_pypi_metadata.rstrip("/"), sections_populate=args.sections_populate, from_local_sdist=from_local_sdist, extras_require_test=args.extras_require_test, diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index 1bed893b7..531a35f34 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -247,14 +247,14 @@ def get_pypi_metadata(config: Configuration) -> dict: """ print_msg("Recovering metadata from pypi...") if config.version: - url_pypi = config.url_pypi_metadata.format( + url_pypi_metadata = config.url_pypi_metadata.format( pkg_name=f"{config.name}/{config.version}" ) else: log.info(f"Version for {config.name} not specified.\nGetting the latest one.") - url_pypi = config.url_pypi_metadata.format(pkg_name=config.name) + url_pypi_metadata = config.url_pypi_metadata.format(pkg_name=config.name) - metadata = requests.get(url=url_pypi, timeout=5) + metadata = requests.get(url=url_pypi_metadata, timeout=5) if metadata.status_code != 200: raise requests.HTTPError( f"It was not possible to recover package metadata for {config.name}.\n" @@ -288,7 +288,7 @@ def get_pypi_metadata(config: Configuration) -> dict: "url": info.get("home_page"), "license": info.get("license"), "source": { - "url": "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/" + "url": config.url_pypi + "/packages/source/{{ name[0] }}/{{ name }}/" f"{get_url_filename(metadata)}", "sha256": get_sha256_from_pypi_metadata(metadata), }, diff --git a/tests/cli/test_cli_cmds.py b/tests/cli/test_cli_cmds.py index 009c64af0..49d473899 100644 --- a/tests/cli/test_cli_cmds.py +++ b/tests/cli/test_cli_cmds.py @@ -76,6 +76,7 @@ def test_change_pypi_url(mocker): "pytest=5.3.2", is_strict_cf=False, download=False, + url_pypi="https://pypi.org", url_pypi_metadata="http://url_pypi.com/abc", sections_populate=None, from_local_sdist=False, From 0d1a1cc7283dec8e5a53663506f3b919a1ccce44 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 19:32:29 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- grayskull/config.py | 4 +++- grayskull/main.py | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/grayskull/config.py b/grayskull/config.py index d0068a03f..e9df19ca3 100644 --- a/grayskull/config.py +++ b/grayskull/config.py @@ -108,7 +108,9 @@ def get_py_version_available( def __post_init__(self): if not self.url_pypi_metadata.endswith("/{pkg_name}/json"): - self.url_pypi_metadata = self.url_pypi_metadata.rstrip("/") + "/{pkg_name}/json" + self.url_pypi_metadata = ( + self.url_pypi_metadata.rstrip("/") + "/{pkg_name}/json" + ) if self.from_local_sdist: self.local_sdist = self.local_sdist or self.name pkg_repo, pkg_name, pkg_version = parse_pkg_name_version(self.name) diff --git a/grayskull/main.py b/grayskull/main.py index 1cf9c31e3..dbef4143c 100644 --- a/grayskull/main.py +++ b/grayskull/main.py @@ -312,20 +312,28 @@ def generate_recipes_from_list(list_pkgs, args): try: # TODO: Remove before 3.0 release if args.url_pypi_metadata_deprecated and args.url_pypi_metadata: - raise RuntimeError("--pypi-url is deprecated in favor of --pypi-url-metadata and may not be passed in conjunction with --pypi-url-metadata") + raise RuntimeError( + "--pypi-url is deprecated in favor of --pypi-url-metadata and may not be passed in conjunction with --pypi-url-metadata" + ) # TODO: Remove before 3.0 release if args.url_pypi_metadata_deprecated is not None: - logging.warning("--pypi-url is deprecated; use --pypi-url-metadata instead") + logging.warning( + "--pypi-url is deprecated; use --pypi-url-metadata instead" + ) args.url_pypi_metadata = args.url_pypi_metadata_deprecated - + # If a PYPI mirror is selected, but the metadata URL is not # explicitly passed, assume the mirror can handle the standard # metadata endpoint and coerce the metadata URL appropriately in a # way that respects the DEFAULT settings from config. - if (args.url_pypi_mirror.rstrip("/") != DEFAULT_PYPI_URL) and (args.url_pypi_metadata.rstrip("/") == DEFAULT_PYPI_META_URL): - args.url_pypi_metadata = DEFAULT_PYPI_META_URL.replace(DEFAULT_PYPI_URL, args.url_pypi_mirror.rstrip("/")) - + if (args.url_pypi_mirror.rstrip("/") != DEFAULT_PYPI_URL) and ( + args.url_pypi_metadata.rstrip("/") == DEFAULT_PYPI_META_URL + ): + args.url_pypi_metadata = DEFAULT_PYPI_META_URL.replace( + DEFAULT_PYPI_URL, args.url_pypi_mirror.rstrip("/") + ) + recipe, config = create_python_recipe( pkg_name, is_strict_cf=args.is_strict_conda_forge, From c0c4ec7f9b1cc504cddb0bb378cd2719334841ad Mon Sep 17 00:00:00 2001 From: Faustin Carter Date: Fri, 24 May 2024 12:49:26 -0700 Subject: [PATCH 3/3] Line length --- grayskull/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/grayskull/main.py b/grayskull/main.py index dbef4143c..542ea90fb 100644 --- a/grayskull/main.py +++ b/grayskull/main.py @@ -164,7 +164,10 @@ def init_parser(): "--pypi-metadata-url", default=DEFAULT_PYPI_META_URL, dest="url_pypi_metadata", - help="Pypi url server metadata endpoint; will be appended with '{pkgname}/json'", + help=( + "Pypi url server metadata endpoint;" + + "will be appended with '{pkgname}/json'" + ), ) pypi_parser.add_argument( "--pypi-mirror-url", @@ -313,7 +316,8 @@ def generate_recipes_from_list(list_pkgs, args): # TODO: Remove before 3.0 release if args.url_pypi_metadata_deprecated and args.url_pypi_metadata: raise RuntimeError( - "--pypi-url is deprecated in favor of --pypi-url-metadata and may not be passed in conjunction with --pypi-url-metadata" + "--pypi-url is deprecated in favor of --pypi-url-metadata " + + "and may not be passed in conjunction with --pypi-url-metadata" ) # TODO: Remove before 3.0 release