feat: add --store to load a schema store

pyproject.toml

@@ -17,6 +17,6 @@ warn_redundant_casts = true
 warn_unused_ignores = true
 
 [tool.pytest.ini_options]
-addopts = ["--cov", "validate_pyproject", "--cov-report", "term-missing", "--verbose"]
+addopts = ["--cov", "validate_pyproject", "--cov-report", "term-missing", "--verbose", "--strict-markers"]
 norecursedirs = ["dist", "build", ".tox"]
 testpaths = ["tests"]

src/validate_pyproject/cli.py

@@ -31,7 +31,7 @@
 from .errors import ValidationError
 from .plugins import PluginWrapper
 from .plugins import list_from_entry_points as list_plugins_from_entry_points
-from .remote import RemotePlugin
+from .remote import RemotePlugin, load_store
 
 _logger = logging.getLogger(__package__)
 T = TypeVar("T", bound=NamedTuple)
@@ -106,13 +106,18 @@ def critical_logging():
         dest="tool",
         help="External tools file/url(s) to load, of the form name=URL#path",
     ),
+    "store": dict(
+        flags=("--store",),
+        help="Load a pyproject.toml file and read all the $ref's into tools",
+    ),
 }
 
 
 class CliParams(NamedTuple):
     input_file: List[io.TextIOBase]
     plugins: List[PluginWrapper]
     tool: List[str]
+    store: str
     loglevel: int = logging.WARNING
     dump_json: bool = False
 
@@ -156,6 +161,7 @@ def parse_args(
     disabled = params.pop("disable", ())
     params["plugins"] = select_plugins(plugins, enabled, disabled)
     params["tool"] = params["tool"] or []
+    params["store"] = params["store"] or ""
     return params_class(**params)  # type: ignore[call-overload]
 
 
@@ -215,6 +221,8 @@ def run(args: Sequence[str] = ()):
     params: CliParams = parse_args(args, plugins)
     setup_logging(params.loglevel)
     tool_plugins = [RemotePlugin.from_str(t) for t in params.tool]
+    if params.store:
+        tool_plugins.extend(load_store(params.store))
     validator = Validator(params.plugins, extra_plugins=tool_plugins)
 
     exceptions = _ExceptionGroup()

src/validate_pyproject/pre_compile/cli.py

@@ -12,7 +12,7 @@
 from .. import cli
 from ..plugins import PluginWrapper
 from ..plugins import list_from_entry_points as list_plugins_from_entry_points
-from ..remote import RemotePlugin
+from ..remote import RemotePlugin, load_store
 from . import pre_compile
 
 if sys.platform == "win32":  # pragma: no cover
@@ -65,6 +65,10 @@ def JSON_dict(name: str, value: str):
         dest="tool",
         help="External tools file/url(s) to load, of the form name=URL#path",
     ),
+    "store": dict(
+        flags=("--store",),
+        help="Load a pyproject.toml file and read all the $ref's into tools",
+    ),
 }
 
 
@@ -82,6 +86,7 @@ class CliParams(NamedTuple):
     replacements: Mapping[str, str] = MappingProxyType({})
     loglevel: int = logging.WARNING
     tool: Sequence[str] = ()
+    store: str = ""
 
 
 def parser_spec(plugins: Sequence[PluginWrapper]) -> Dict[str, dict]:
@@ -101,6 +106,8 @@ def run(args: Sequence[str] = ()):
     cli.setup_logging(prms.loglevel)
 
     tool_plugins = [RemotePlugin.from_str(t) for t in prms.tool]
+    if prms.store:
+        tool_plugins.extend(load_store(prms.store))
 
     pre_compile(
         prms.output_dir,

src/validate_pyproject/remote.py

@@ -1,10 +1,11 @@
 import io
 import json
+import logging
 import sys
 import typing
 import urllib.parse
 import urllib.request
-from typing import Tuple
+from typing import Generator, Tuple
 
 from . import errors
 from .types import Schema
@@ -26,7 +27,10 @@ def open_url(url: str) -> io.StringIO:
             return io.StringIO(response.read().decode("utf-8"))
 
 
-__all__ = ["RemotePlugin"]
+__all__ = ["RemotePlugin", "load_store"]
+
+
+_logger = logging.getLogger(__name__)
 
 
 def load_from_uri(tool_uri: str) -> Tuple[str, Schema]:
@@ -42,18 +46,45 @@ def load_from_uri(tool_uri: str) -> Tuple[str, Schema]:
 
 
 class RemotePlugin:
-    def __init__(self, tool: str, url: str):
+    def __init__(self, *, tool: str, schema: Schema, fragment: str = ""):
         self.tool = tool
-        self.fragment, self.schema = load_from_uri(url)
+        self.schema = schema
+        self.fragment = fragment
         self.id = self.schema["$id"]
         self.help_text = f"{tool} <external>"
 
+    @classmethod
+    def from_url(cls, tool: str, url: str):
+        fragment, schema = load_from_uri(url)
+        return cls(tool=tool, schema=schema, fragment=fragment)
+
     @classmethod
     def from_str(cls, tool_url: str) -> "Self":
         tool, _, url = tool_url.partition("=")
         if not url:
             raise errors.URLMissingTool(tool)
-        return cls(tool, url)
+        return cls.from_url(tool, url)
+
+
+def load_store(pyproject_url: str) -> Generator[RemotePlugin, None, None]:
+    """
+    Takes a URL / Path and loads the tool table, assuming it is a set of ref's.
+    Currently ignores "inline" sections. This is the format that SchemaStore
+    (https://json.schemastore.org/pyproject.json) is in.
+    """
+
+    fragment, contents = load_from_uri(pyproject_url)
+    if fragment:
+        _logger.error(f"Must not be called with a fragment, got {fragment!r}")
+    table = contents["properties"]["tool"]["properties"]
+    for tool, info in table.items():
+        if tool in {"setuptools", "distutils"}:
+            pass  # built-in
+        elif "$ref" in info:
+            _logger.info(f"Loading {tool} from store: {pyproject_url}")
+            yield RemotePlugin.from_url(tool, info["$ref"])
+        else:
+            _logger.warning(f"{tool!r} does not contain $ref")
 
 
 if typing.TYPE_CHECKING:

tests/examples/store/example.toml

@@ -0,0 +1,86 @@
+[tool.ruff]
+src = ["src"]
+
+[tool.ruff.lint]
+extend-select = [
+  "B",           # flake8-bugbear
+  "I",           # isort
+  "ARG",         # flake8-unused-arguments
+  "C4",          # flake8-comprehensions
+  "EM",          # flake8-errmsg
+  "ICN",         # flake8-import-conventions
+  "G",           # flake8-logging-format
+  "PGH",         # pygrep-hooks
+  "PIE",         # flake8-pie
+  "PL",          # pylint
+  "PT",          # flake8-pytest-style
+  "PTH",         # flake8-use-pathlib
+  "RET",         # flake8-return
+  "RUF",         # Ruff-specific
+  "SIM",         # flake8-simplify
+  "T20",         # flake8-print
+  "UP",          # pyupgrade
+  "YTT",         # flake8-2020
+  "EXE",         # flake8-executable
+  "NPY",         # NumPy specific rules
+  "PD",          # pandas-vet
+  "FURB",        # refurb
+  "PYI",         # flake8-pyi
+]
+ignore = [
+  "PLR",    # Design related pylint codes
+]
+typing-modules = ["mypackage._compat.typing"]
+isort.required-imports = ["from __future__ import annotations"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["T20"]
+
+
+[tool.cibuildwheel]
+build = "*"
+skip = ""
+test-skip = ""
+
+archs = ["auto"]
+build-frontend = "default"
+config-settings = {}
+dependency-versions = "pinned"
+environment = {}
+environment-pass = []
+build-verbosity = 0
+
+before-all = ""
+before-build = ""
+repair-wheel-command = ""
+
+test-command = ""
+before-test = ""
+test-requires = []
+test-extras = []
+
+container-engine = "docker"
+
+manylinux-x86_64-image = "manylinux2014"
+manylinux-i686-image = "manylinux2014"
+manylinux-aarch64-image = "manylinux2014"
+manylinux-ppc64le-image = "manylinux2014"
+manylinux-s390x-image = "manylinux2014"
+manylinux-pypy_x86_64-image = "manylinux2014"
+manylinux-pypy_i686-image = "manylinux2014"
+manylinux-pypy_aarch64-image = "manylinux2014"
+
+musllinux-x86_64-image = "musllinux_1_1"
+musllinux-i686-image = "musllinux_1_1"
+musllinux-aarch64-image = "musllinux_1_1"
+musllinux-ppc64le-image = "musllinux_1_1"
+musllinux-s390x-image = "musllinux_1_1"
+
+
+[tool.cibuildwheel.linux]
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
+
+[tool.cibuildwheel.macos]
+repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"
+
+[tool.cibuildwheel.windows]

tests/examples/store/test_config.json

@@ -0,0 +1,3 @@
+{
+    "store": "https://json.schemastore.org/pyproject.json"
+}

tests/helpers.py

@@ -1,6 +1,9 @@
+import functools
 import json
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List, Union
+
+from validate_pyproject.remote import RemotePlugin, load_store
 
 HERE = Path(__file__).parent.resolve()
 
@@ -13,9 +16,31 @@ def error_file(p: Path) -> Path:
         raise FileNotFoundError(f"No error file found for {p}") from None
 
 
-def get_test_config(example: Path) -> Dict[str, str]:
+def get_test_config(example: Path) -> Dict[str, Union[str, Dict[str, str]]]:
     test_config = example.with_name("test_config.json")
     if test_config.is_file():
         with test_config.open(encoding="utf-8") as f:
             return json.load(f)
     return {}
+
+
+@functools.lru_cache(maxsize=None)
+def get_tools(example: Path) -> List[RemotePlugin]:
+    config = get_test_config(example)
+    tools: Dict[str, str] = config.get("tools", {})
+    load_tools = [RemotePlugin.from_url(k, v) for k, v in tools.items()]
+    store: str = config.get("store", "")
+    if store:
+        load_tools.extend(load_store(store))
+    return load_tools
+
+
+@functools.lru_cache(maxsize=None)
+def get_tools_as_args(example: Path) -> List[str]:
+    config = get_test_config(example)
+    tools: Dict[str, str] = config.get("tools", {})
+    load_tools = [f"--tool={k}={v}" for k, v in tools.items()]
+    store: str = config.get("store", "")
+    if store:
+        load_tools.append(f"--store={store}")
+    return load_tools

tests/invalid-examples/store/cibw-overrides-noaction.errors.txt

@@ -0,0 +1 @@
+`tool.cibuildwheel.overrides[0]` must contain at least 2 properties

tests/invalid-examples/store/cibw-overrides-noaction.toml

@@ -0,0 +1,5 @@
+[tool.cibuildwheel]
+build = "*"
+
+[[tool.cibuildwheel.overrides]]
+select = "cp312-*"

tests/invalid-examples/store/cibw-overrides-noselect.errors.txt

@@ -0,0 +1 @@
+`tool.cibuildwheel.overrides[0]` must contain ['select'] properties

tests/invalid-examples/store/cibw-overrides-noselect.toml

@@ -0,0 +1,6 @@
+[tool.cibuildwheel]
+build = "*"
+
+[[tool.cibuildwheel.overrides]]
+test-command = "pytest"
+test-extras = "test"

tests/invalid-examples/store/cibw-unknown-option.errors.txt

@@ -0,0 +1 @@
+`tool.cibuildwheel` must not contain {'no-a-read-option'} properties

tests/invalid-examples/store/cibw-unknown-option.toml

@@ -0,0 +1,2 @@
+[tool.cibuildwheel]
+no-a-read-option = "error"

tests/invalid-examples/store/ruff-badcode.errors.txt

@@ -0,0 +1 @@
+`tool.ruff.lint` cannot be validated by any definition

tests/invalid-examples/store/ruff-badcode.toml

@@ -0,0 +1,2 @@
+[tool.ruff.lint]
+extend-select = ["NOTACODE"]

tests/invalid-examples/store/ruff-unknown.errors.txt

@@ -0,0 +1 @@
+`tool.ruff` must not contain {'not-a-real-option'} properties

tests/invalid-examples/store/ruff-unknown.toml

@@ -0,0 +1,2 @@
+[tool.ruff]
+not-a-real-option = true

tests/invalid-examples/store/test_config.json

@@ -0,0 +1,3 @@
+{
+    "store": "https://json.schemastore.org/pyproject.json"
+}

tests/test_examples.py

@@ -6,30 +6,26 @@
 from validate_pyproject import _tomllib as tomllib
 from validate_pyproject import api, cli
 from validate_pyproject.error_reporting import ValidationError
-from validate_pyproject.remote import RemotePlugin
 
-from .helpers import error_file, get_test_config
+from .helpers import error_file, get_tools, get_tools_as_args
 
 
 def test_examples_api(example: Path) -> None:
-    tools = get_test_config(example).get("tools", {})
-    load_tools = [RemotePlugin.from_str(f"{k}={v}") for k, v in tools.items()]
+    load_tools = get_tools(example)
 
     toml_equivalent = tomllib.loads(example.read_text())
     validator = api.Validator(extra_plugins=load_tools)
     assert validator(toml_equivalent) is not None
 
 
 def test_examples_cli(example: Path) -> None:
-    tools = get_test_config(example).get("tools", {})
-    args = [f"--tool={k}={v}" for k, v in tools.items()]
+    args = get_tools_as_args(example)
 
     assert cli.run(["--dump-json", str(example), *args]) == 0  # no errors
 
 
 def test_invalid_examples_api(invalid_example: Path) -> None:
-    tools = get_test_config(invalid_example).get("tools", {})
-    load_tools = [RemotePlugin.from_str(f"{k}={v}") for k, v in tools.items()]
+    load_tools = get_tools(invalid_example)
 
     expected_error = error_file(invalid_example).read_text("utf-8")
     toml_equivalent = tomllib.loads(invalid_example.read_text())
@@ -44,8 +40,7 @@ def test_invalid_examples_api(invalid_example: Path) -> None:
 
 
 def test_invalid_examples_cli(invalid_example: Path, caplog) -> None:
-    tools = get_test_config(invalid_example).get("tools", {})
-    args = [f"--tool={k}={v}" for k, v in tools.items()]
+    args = get_tools_as_args(invalid_example)
 
     caplog.set_level(logging.DEBUG)
     expected_error = error_file(invalid_example).read_text("utf-8")