From 098b2b18338360f8e2d1d18939e130a47a64447c Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 19 Apr 2023 10:18:04 +0200 Subject: [PATCH 1/2] global generators in cache --- conan/api/subapi/install.py | 2 +- conans/client/cache/cache.py | 4 ++ conans/client/generators/__init__.py | 26 ++++++- conans/client/installer.py | 4 +- .../test_custom_global_generators.py | 69 +++++++++++++++++++ 5 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 conans/test/integration/generators/test_custom_global_generators.py diff --git a/conan/api/subapi/install.py b/conan/api/subapi/install.py index 14e94f172bc..71ba81889cb 100644 --- a/conan/api/subapi/install.py +++ b/conan/api/subapi/install.py @@ -69,4 +69,4 @@ def install_consumer(self, deps_graph, generators=None, source_folder=None, outp conanfile.generators = list(set(conanfile.generators).union(generators or [])) app = ConanApp(self.conan_api.cache_folder) - write_generators(conanfile, app.hook_manager) + write_generators(conanfile, app) diff --git a/conans/client/cache/cache.py b/conans/client/cache/cache.py index ea45015f5f6..30c7887217e 100644 --- a/conans/client/cache/cache.py +++ b/conans/client/cache/cache.py @@ -188,6 +188,10 @@ def settings_path(self): def custom_commands_path(self): return os.path.join(self.cache_folder, EXTENSIONS_FOLDER, CUSTOM_COMMANDS_FOLDER) + @property + def custom_generators_path(self): + return os.path.join(self.cache_folder, EXTENSIONS_FOLDER, "generators") + @property def plugins_path(self): return os.path.join(self.cache_folder, EXTENSIONS_FOLDER, PLUGINS_FOLDER) diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index 29a7c5252f1..ad9f8b88771 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -1,3 +1,4 @@ +import inspect import os import traceback import importlib @@ -45,10 +46,30 @@ def _get_generator_class(generator_name): f"inside module {generator_class}") from e -def write_generators(conanfile, hook_manager): +def load_cache_generators(path): + from conans.client.loader import load_python_file + result = {} # Name of the generator: Class + if not os.path.isdir(path): + return result + for f in os.listdir(path): + if not f.endswith(".py") or f.startswith("_"): + continue + full_path = os.path.join(path, f) + mod, _ = load_python_file(full_path) + for name, value in inspect.getmembers(mod): + if inspect.isclass(value) and not name.startswith("_"): + result = {name: value} + return result + + +def write_generators(conanfile, app): new_gen_folder = conanfile.generators_folder _receive_conf(conanfile) + hook_manager = app.hook_manager + cache = app.cache + # TODO: Optimize this, so the global generators are not loaded every call to write_generators + global_generators = load_cache_generators(cache.custom_generators_path) hook_manager.execute("pre_generate", conanfile=conanfile) if conanfile.generators: @@ -60,7 +81,8 @@ def write_generators(conanfile, hook_manager): conanfile.generators = [] try: for generator_name in old_generators: - generator_class = _get_generator_class(generator_name) + global_generator = global_generators.get(generator_name) + generator_class = global_generator or _get_generator_class(generator_name) if generator_class: try: generator = generator_class(conanfile) diff --git a/conans/client/installer.py b/conans/client/installer.py index 2f0de5f4063..64a60d3761f 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -88,7 +88,7 @@ def _copy_sources(conanfile, source_folder, build_folder): raise ConanException("%s\nError copying sources to build folder" % msg) def _build(self, conanfile, pref): - write_generators(conanfile, self._hook_manager) + write_generators(conanfile, self._app) try: run_build_method(conanfile, self._hook_manager) @@ -351,7 +351,7 @@ def _handle_node_editable(self, install_node): output = conanfile.output output.info("Rewriting files of editable package " "'{}' at '{}'".format(conanfile.name, conanfile.generators_folder)) - write_generators(conanfile, self._hook_manager) + write_generators(conanfile, self._app) if node.binary == BINARY_EDITABLE_BUILD: run_build_method(conanfile, self._hook_manager) diff --git a/conans/test/integration/generators/test_custom_global_generators.py b/conans/test/integration/generators/test_custom_global_generators.py new file mode 100644 index 00000000000..ca34a1671ed --- /dev/null +++ b/conans/test/integration/generators/test_custom_global_generators.py @@ -0,0 +1,69 @@ +import os +import textwrap + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.tools import TestClient +from conans.util.files import save + + +def test_custom_global_generator(): + c = TestClient() + generator = textwrap.dedent(""" + class MyCustomGenerator: + def __init__(self, conanfile): + self._conanfile = conanfile + def generate(self): + pkg = self._conanfile.dependencies["pkg"].ref + self._conanfile.output.info(f"DEP: {pkg}!!") + """) + save(os.path.join(c.cache.custom_generators_path, "mygen.py"), generator) + conanfile = textwrap.dedent(""" + [requires] + pkg/0.1 + + [generators] + MyCustomGenerator + """) + c.save({"pkg/conanfile.py": GenConanfile("pkg", "0.1"), + "conanfile.txt": conanfile}) + c.run("create pkg") + c.run("install .") + assert "conanfile.txt: Generator 'MyCustomGenerator' calling 'generate()'" in c.out + assert "conanfile.txt: DEP: pkg/0.1!!" in c.out + + # By CLI also works + conanfile = textwrap.dedent(""" + [requires] + pkg/0.1 + """) + c.save({"conanfile.txt": conanfile}) + c.run("install . -g MyCustomGenerator") + assert "conanfile.txt: Generator 'MyCustomGenerator' calling 'generate()'" in c.out + assert "conanfile.txt: DEP: pkg/0.1!!" in c.out + + +def test_custom_global_generator_imports(): + """ + our custom generator can use python imports + """ + c = TestClient() + generator = textwrap.dedent(""" + from _myfunc import mygenerate + + class MyCustomGenerator: + def __init__(self, conanfile): + self._conanfile = conanfile + def generate(self): + mygenerate(self._conanfile) + """) + myaux = textwrap.dedent(""" + def mygenerate(conanfile): + conanfile.output.info("MYGENERATE WORKS!!") + """) + save(os.path.join(c.cache.custom_generators_path, "mygen.py"), generator) + save(os.path.join(c.cache.custom_generators_path, "_myfunc.py"), myaux) + + c.save({"conanfile.txt": ""}) + c.run("install . -g MyCustomGenerator") + assert "conanfile.txt: Generator 'MyCustomGenerator' calling 'generate()'" in c.out + assert "conanfile.txt: MYGENERATE WORKS!!" in c.out From ed6074ba9a4b6beb0a1ef11dcd324617e7208f96 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 19 Apr 2023 10:21:14 +0200 Subject: [PATCH 2/2] add test --- .../generators/test_custom_global_generators.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/conans/test/integration/generators/test_custom_global_generators.py b/conans/test/integration/generators/test_custom_global_generators.py index ca34a1671ed..e59653ad4f3 100644 --- a/conans/test/integration/generators/test_custom_global_generators.py +++ b/conans/test/integration/generators/test_custom_global_generators.py @@ -41,6 +41,18 @@ def generate(self): assert "conanfile.txt: Generator 'MyCustomGenerator' calling 'generate()'" in c.out assert "conanfile.txt: DEP: pkg/0.1!!" in c.out + # In conanfile.py also works + conanfile = textwrap.dedent(""" + from conan import ConanFile + class MyPkg(ConanFile): + requires = "pkg/0.1" + generators = "MyCustomGenerator" + """) + c.save({"conanfile.py": conanfile}, clean_first=True) + c.run("install . ") + assert "conanfile.py: Generator 'MyCustomGenerator' calling 'generate()'" in c.out + assert "conanfile.py: DEP: pkg/0.1!!" in c.out + def test_custom_global_generator_imports(): """