Initial support for pyproject.toml (sagemath#425)

* Initial support for pyproject.toml

grayskull/config.py

@@ -101,10 +101,10 @@ def get_py_version_available(
 
     def __post_init__(self):
         if self.from_local_sdist:
-            self.local_sdist = self.name
+            self.local_sdist = self.local_sdist or self.name
         if self.url_pypi_metadata != "https://pypi.org/pypi/{pkg_name}/json":
-            preffix = "" if self.url_pypi_metadata.endswith("/") else "/"
-            self.url_pypi_metadata += f"{preffix}{{pkg_name}}/json"
+            prefix = "" if self.url_pypi_metadata.endswith("/") else "/"
+            self.url_pypi_metadata += f"{prefix}{{pkg_name}}/json"
         pkg_repo, pkg_name, pkg_version = parse_pkg_name_version(self.name)
         if pkg_repo:
             prefix = "" if pkg_repo.endswith("/") else "/"

grayskull/strategy/py_base.py

@@ -3,9 +3,11 @@
 import re
 import shutil
 import sys
+from collections import defaultdict
 from contextlib import contextmanager
 from copy import deepcopy
 from distutils import core
+from glob import glob
 from pathlib import Path
 from subprocess import check_output
 from tempfile import mkdtemp
@@ -23,6 +25,7 @@
 from grayskull.cli.stdout import manage_progressbar, print_msg
 from grayskull.config import Configuration
 from grayskull.license.discovery import ShortLicense, search_license_file
+from grayskull.strategy.py_toml import get_all_toml_info
 from grayskull.utils import (
     PyVer,
     get_vendored_dependencies,
@@ -683,6 +686,54 @@ def download_sdist_pkg(sdist_url: str, dest: str, name: Optional[str] = None):
                 bar.update(min(progress_val, total_size))
 
 
+def merge_deps_toml_setup(setup_deps: list, toml_deps: list) -> list:
+    re_split = re.compile(r"\s+|>|=|<|~|!")
+    all_deps = defaultdict(list)
+    for dep in toml_deps + setup_deps:
+        if dep.strip():
+            dep_name = re_split.split(dep)[0]
+            if dep_name not in all_deps:
+                if dep_name.replace("_", "-") in all_deps:
+                    dep_name = dep_name.replace("_", "-")
+                elif dep_name.replace("-", "_") in all_deps:
+                    dep_name = dep_name.replace("-", "_")
+            all_deps[dep_name].append(dep)
+
+    return [deps[0] for deps in all_deps.values()]
+
+
+def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) -> dict:
+    setup_metadata = defaultdict(dict, setup_metadata)
+    if not pyproject_metadata:
+        return setup_metadata
+    if pyproject_metadata["about"]["license"]:
+        setup_metadata["license"] = pyproject_metadata["about"]["license"]
+    if pyproject_metadata["about"]["summary"]:
+        setup_metadata["summary"] = pyproject_metadata["about"]["summary"]
+    if pyproject_metadata["about"]["home"]:
+        setup_metadata["projects_url"]["Homepage"] = pyproject_metadata["about"]["home"]
+    if pyproject_metadata["build"]["entry_points"]:
+        setup_metadata["entry_points"]["console_scripts"] = pyproject_metadata["build"][
+            "entry_points"
+        ]
+    if pyproject_metadata["test"]["requires"]:
+        setup_metadata["extras_require"]["testing"] = merge_deps_toml_setup(
+            setup_metadata["extras_require"].get("testing", []),
+            pyproject_metadata["test"]["requires"],
+        )
+    if pyproject_metadata["requirements"]["host"]:
+        setup_metadata["setup_requires"] = merge_deps_toml_setup(
+            setup_metadata.get("setup_requires", []),
+            pyproject_metadata["requirements"]["host"],
+        )
+    if pyproject_metadata["requirements"]["run"]:
+        setup_metadata["install_requires"] = merge_deps_toml_setup(
+            setup_metadata.get("install_requires", []),
+            pyproject_metadata["requirements"]["run"],
+        )
+    return setup_metadata
+
+
 def get_sdist_metadata(
     sdist_url: str, config: Configuration, with_source: bool = False
 ) -> dict:
@@ -705,6 +756,17 @@ def get_sdist_metadata(
             config.files_to_copy.append(path_pkg)
     log.debug(f"Unpacking {path_pkg} to {temp_folder}")
     shutil.unpack_archive(path_pkg, temp_folder)
+
+    print_msg("Checking for pyproject.toml")
+    pyproject_toml = glob(f"{temp_folder}/**/pyproject.toml", recursive=True)
+    pyproject_metadata = {}
+    if pyproject_toml:
+        pyproject_toml = Path(pyproject_toml[0])
+        print_msg(f"pyproject.toml found in {pyproject_toml}")
+        pyproject_metadata = get_all_toml_info(pyproject_toml)
+    else:
+        print_msg("pyproject.toml not found.")
+
     print_msg("Recovering information from setup.py")
     with injection_distutils(temp_folder) as metadata:
         metadata["sdist_path"] = temp_folder
@@ -725,7 +787,7 @@ def get_sdist_metadata(
         dist = UnpackedSDist(path_pkg_info[0].parent)
         for key in ("name", "version", "summary", "author"):
             metadata[key] = getattr(dist, key, None)
-    return metadata
+    return merge_setup_toml_metadata(metadata, pyproject_metadata)
 
 
 def ensure_pep440_in_req_list(list_req: List[str]) -> List[str]:

grayskull/strategy/py_toml.py

@@ -0,0 +1,42 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Union
+
+import tomli
+
+from grayskull.utils import nested_dict
+
+
+def get_all_toml_info(path_toml: Union[Path, str]) -> dict:
+    with open(path_toml, "rb") as f:
+        toml_metadata = tomli.load(f)
+    toml_metadata = defaultdict(dict, toml_metadata)
+    metadata = nested_dict()
+
+    metadata["requirements"]["host"] = toml_metadata["build-system"].get("requires", [])
+    metadata["requirements"]["run"] = toml_metadata["project"].get("dependencies", [])
+    license = toml_metadata["project"].get("license")
+    if isinstance(license, dict):
+        license = license.get("text", "")
+    metadata["about"]["license"] = license
+    metadata["test"]["requires"] = (
+        toml_metadata["project"].get("optional-dependencies", {}).get("testing", [])
+    )
+
+    if toml_metadata["project"].get("requires-python"):
+        py_constrain = f"python {toml_metadata['project']['requires-python']}"
+        metadata["requirements"]["host"].append(py_constrain)
+        metadata["requirements"]["run"].append(py_constrain)
+
+    if toml_metadata["project"].get("scripts"):
+        metadata["build"]["entry_points"] = []
+        for entry_name, entry_path in (
+            toml_metadata["project"].get("scripts", {}).items()
+        ):
+            metadata["build"]["entry_points"].append(f"{entry_name} = {entry_path}")
+    all_urls = toml_metadata["project"].get("urls")
+    if all_urls:
+        metadata["about"]["dev_url"] = all_urls.get("Source", None)
+        metadata["about"]["home"] = all_urls.get("Homepage", None)
+    metadata["about"]["summary"] = toml_metadata["project"].get("description")
+    return metadata

grayskull/utils.py

@@ -3,7 +3,7 @@
 import logging
 import os
 import re
-from collections import namedtuple
+from collections import defaultdict, namedtuple
 from difflib import SequenceMatcher
 from functools import lru_cache
 from glob import glob
@@ -259,3 +259,7 @@ def merge_dict_of_lists_item(destination: dict, add: dict, key: str) -> None:
         merge_list_item(sub_destination, sub_add, sub_key)
     if sub_destination:
         destination[key] = sub_destination
+
+
+def nested_dict():
+    return defaultdict(nested_dict)

tests/data/pyproject/tox.toml

@@ -0,0 +1,162 @@
+[build-system]
+build-backend = "hatchling.build"
+requires = ["hatchling>=1.12.2", "hatch-vcs>=0.3"]
+
+[project]
+name = "tox"
+description = "tox is a generic virtualenv management and test command line tool"
+readme.file = "README.md"
+readme.content-type = "text/markdown"
+keywords = ["virtual", "environments", "isolated", "testing"]
+license = "MIT"
+urls.Homepage = "http://tox.readthedocs.org"
+urls.Documentation = "https://tox.wiki"
+urls.Source = "https://github.com/tox-dev/tox"
+urls.Tracker = "https://github.com/tox-dev/tox/issues"
+urls."Release Notes" = "https://tox.wiki/en/latest/changelog.html"
+authors = [{ name = "Bernát Gábor", email = "gaborjbernat@gmail.com" }]
+maintainers = [
+  { name = "Anthony Sottile", email = "asottile@umich.edu" },
+  { name = "Bernát Gábor", email = "gaborjbernat@gmail.com" },
+  { name = "Jürgen Gmach", email = "juergen.gmach@googlemail.com" },
+  { name = "Oliver Bestwalter", email = "oliver@bestwalter.de" },
+]
+requires-python = ">=3.7"
+dependencies = [
+  "cachetools>=5.2.1",
+  "chardet>=5.1",
+  "colorama>=0.4.6",
+  "packaging>=23",
+  "platformdirs>=2.6.2",
+  "pluggy>=1",
+  "pyproject-api>=1.5",
+  'tomli>=2.0.1; python_version < "3.11"',
+  "virtualenv>=20.17.1",
+  "filelock>=3.9",
+  'importlib-metadata>=6; python_version < "3.8"',
+  'typing-extensions>=4.4; python_version < "3.8"',
+]
+optional-dependencies.docs = [
+  "furo>=2022.12.7",
+  "sphinx>=6.1.3",
+  "sphinx-argparse-cli>=1.11",
+  "sphinx-autodoc-typehints>=1.20.1",
+  "sphinx-copybutton>=0.5.1",
+  "sphinx-inline-tabs>=2022.1.2b11",
+  "sphinxcontrib-towncrier>=0.2.1a0",
+  "towncrier>=22.12",
+]
+optional-dependencies.testing = [
+  "build[virtualenv]>=0.9",
+  "covdefaults>=2.2.2",
+  "devpi-process>=0.3",
+  "diff-cover>=7.3",
+  "distlib>=0.3.6",
+  "flaky>=3.7",
+  "hatch-vcs>=0.3",
+  "hatchling>=1.12.2",
+  "psutil>=5.9.4",
+  "pytest>=7.2",
+  "pytest-cov>=4",
+  "pytest-mock>=3.10",
+  "pytest-xdist>=3.1",
+  "re-assert>=1.1",
+  "wheel>=0.38.4",
+  "time-machine>=2.8.2; implementation_name != \"pypy\"",
+]
+scripts.tox = "tox.run:run"
+dynamic = ["version"]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Framework :: tox",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: MacOS :: MacOS X",
+  "Operating System :: Microsoft :: Windows",
+  "Operating System :: POSIX",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.7",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Topic :: Software Development :: Libraries",
+  "Topic :: Software Development :: Testing",
+  "Topic :: Utilities",
+]
+
+[tool.hatch]
+build.dev-mode-dirs = ["src"]
+build.hooks.vcs.version-file = "src/tox/version.py"
+build.targets.sdist.include = ["/src", "/tests"]
+version.source = "vcs"
+
+[tool.black]
+line-length = 120
+
+[tool.coverage]
+html.show_contexts = true
+html.skip_covered = false
+paths.source = [
+  "src",
+  ".tox*/*/lib/python*/site-packages",
+  ".tox*/pypy*/site-packages",
+  ".tox*\\*\\Lib\\site-packages",
+  "*/src",
+  "*\\src",
+]
+report.fail_under = 88
+report.omit = ["src/tox/config/cli/for_docs.py", "tests/execute/local_subprocess/bad_process.py", "tests/type_check/*"]
+run.parallel = true
+run.plugins = ["covdefaults"]
+
+[tool.isort]
+known_first_party = ["tox", "tests"]
+profile = "black"
+line_length = 120
+
+[tool.mypy]
+python_version = "3.11"
+show_error_codes = true
+strict = true
+overrides = [
+  { module = [
+    "colorama.*",
+    "coverage.*",
+    "distlib.*",
+    "flaky.*",
+    "importlib_metadata.*",
+    "pluggy.*",
+    "psutil.*",
+    "re_assert.*",
+    "virtualenv.*",
+  ], ignore_missing_imports = true },
+]
+
+[tool.pep8]
+max-line-length = "120"
+
+[tool.flake8]
+max-complexity = 22
+max-line-length = 120
+unused-arguments-ignore-abstract-functions = true
+noqa-require-code = true
+dictionaries = ["en_US", "python", "technical", "django"]
+ignore = [
+  "E203", # whitespace before ':'
+  "W503", #  line break before binary operator
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--tb=auto -ra --showlocals --no-success-flaky-report"
+
+[tool.towncrier]
+name = "tox"
+filename = "docs/changelog.rst"
+directory = "docs/changelog"
+title_format = false
+issue_format = ":issue:`{issue}`"
+template = "docs/changelog/template.jinja2"
+# possible types, all default: feature, bugfix, doc, removal, misc