Config clean command sketch (#17514)

* Clean command sketch

* Make clean/Migrate part of the ConfigAPI

* Test

* Comment

* Basic

* Only reinit for valid apis

* Add reinit test()

* Missing reinit()

* Review comments

* Remove todo

* fix tests

* Migrate is part of the full api now

* Listen to core.cache:storage_path

* Reinit api when config is cleaned

* Reinit conan_api when using --core-conf

* First initial approach for fixed reinit

* Better reinit

* Fix ConanOutput confs

* Dont invalidate reinit global_conf

* Remove useless import

conan/api/conan_api.py

@@ -6,7 +6,6 @@
 from conan.api.subapi.local import LocalAPI
 from conan.api.subapi.lockfile import LockfileAPI
 from conan.api.subapi.workspace import WorkspaceAPI
-from conan import conan_version
 from conan.api.subapi.config import ConfigAPI
 from conan.api.subapi.download import DownloadAPI
 from conan.api.subapi.export import ExportAPI
@@ -19,9 +18,9 @@
 from conan.api.subapi.remove import RemoveAPI
 from conan.api.subapi.search import SearchAPI
 from conan.api.subapi.upload import UploadAPI
-from conans.client.migrations import ClientMigrator
 from conan.errors import ConanException
 from conan.internal.paths import get_conan_user_home
+from conans.client.migrations import ClientMigrator
 from conan.internal.model.version_range import validate_conan_version
 
 
@@ -36,12 +35,11 @@ def __init__(self, cache_folder=None):
         self.workspace = WorkspaceAPI(self)
         self.cache_folder = self.workspace.home_folder() or cache_folder or get_conan_user_home()
         self.home_folder = self.cache_folder  # Lets call it home, deprecate "cache"
+        self.migrate()
 
-        # Migration system
-        migrator = ClientMigrator(self.cache_folder, conan_version)
-        migrator.migrate()
-
+        # This API is depended upon by the subsequent ones, it should be initialized first
         self.config = ConfigAPI(self)
+
         self.remotes = RemotesAPI(self)
         self.command = CommandAPI(self)
         # Search recipes by wildcard and packages filtering by configuration
@@ -60,6 +58,24 @@ def __init__(self, cache_folder=None):
         self.lockfile = LockfileAPI(self)
         self.local = LocalAPI(self)
 
-        required_range_new = self.config.global_conf.get("core:required_conan_version")
-        if required_range_new:
-            validate_conan_version(required_range_new)
+        _check_conan_version(self)
+
+    def reinit(self):
+        self.config.reinit()
+        self.remotes.reinit()
+        self.local.reinit()
+
+        _check_conan_version(self)
+
+    def migrate(self):
+        # Migration system
+        # TODO: A prettier refactoring of migrators would be nice
+        from conan import conan_version
+        migrator = ClientMigrator(self.cache_folder, conan_version)
+        migrator.migrate()
+
+
+def _check_conan_version(conan_api):
+    required_range_new = conan_api.config.global_conf.get("core:required_conan_version")
+    if required_range_new:
+        validate_conan_version(required_range_new)

conan/api/subapi/config.py

@@ -16,18 +16,19 @@
 from conans.client.graph.graph_builder import DepsGraphBuilder
 from conans.client.graph.profile_node_definer import consumer_definer
 from conan.errors import ConanException
-from conan.internal.model.conf import ConfDefinition, BUILT_IN_CONFS
+from conan.internal.model.conf import ConfDefinition, BUILT_IN_CONFS, CORE_CONF_PATTERN
 from conan.internal.model.pkg_type import PackageType
 from conan.api.model import RecipeReference
 from conan.internal.model.settings import Settings
-from conans.util.files import load, save
+from conans.util.files import load, save, rmdir, remove
 
 
 class ConfigAPI:
 
     def __init__(self, conan_api):
         self.conan_api = conan_api
         self._new_config = None
+        self._cli_core_confs = None
 
     def home(self):
         return self.conan_api.cache_folder
@@ -40,6 +41,7 @@ def install(self, path_or_url, verify_ssl, config_type=None, args=None,
         requester = self.conan_api.remotes.requester
         configuration_install(cache_folder, requester, path_or_url, verify_ssl, config_type=config_type, args=args,
                               source_folder=source_folder, target_folder=target_folder)
+        self.conan_api.reinit()
 
     def install_pkg(self, ref, lockfile=None, force=False, remotes=None, profile=None):
         ConanOutput().warning("The 'conan config install-pkg' is experimental",
@@ -100,6 +102,7 @@ def install_pkg(self, ref, lockfile=None, force=False, remotes=None, profile=Non
         config_versions = {ref.split("/", 1)[0]: ref for ref in config_versions}
         config_versions[pkg.pref.ref.name] = pkg.pref.repr_notime()
         save(config_version_file, json.dumps({"config_version": list(config_versions.values())}))
+        self.conan_api.reinit()
         return pkg.pref
 
     def get(self, name, default=None, check_type=None):
@@ -114,11 +117,19 @@ def global_conf(self):
         configuration defined with the new syntax as in profiles, this config will be composed
         to the profile ones and passed to the conanfiles.conf, which can be passed to collaborators
         """
+        # Lazy loading
         if self._new_config is None:
-            cache_folder = self.conan_api.cache_folder
-            self._new_config = self.load_config(cache_folder)
+            self._new_config = ConfDefinition()
+            self._populate_global_conf()
         return self._new_config
 
+    def _populate_global_conf(self):
+        cache_folder = self.conan_api.cache_folder
+        new_config = self.load_config(cache_folder)
+        self._new_config.update_conf_definition(new_config)
+        if self._cli_core_confs is not None:
+            self._new_config.update_conf_definition(self._cli_core_confs)
+
     @staticmethod
     def load_config(home_folder):
         # Do not document yet, keep it private
@@ -191,3 +202,35 @@ def appending_recursive_dict_update(d, u):
             appending_recursive_dict_update(settings, settings_user)
 
         return Settings(settings)
+
+    def clean(self):
+        contents = os.listdir(self.home())
+        packages_folder = self.global_conf.get("core.cache:storage_path") or os.path.join(self.home(), "p")
+        for content in contents:
+            content_path = os.path.join(self.home(), content)
+            if content_path == packages_folder or content == "version.txt":
+                continue
+            ConanOutput().debug(f"Removing {content_path}")
+            if os.path.isdir(content_path):
+                rmdir(content_path)
+            else:
+                remove(content_path)
+        self.conan_api.reinit()
+        # CHECK: This also generates a remotes.json that is not there after a conan profile show?
+        self.conan_api.migrate()
+
+    def set_core_confs(self, core_confs):
+        confs = ConfDefinition()
+        for c in core_confs:
+            if not CORE_CONF_PATTERN.match(c):
+                raise ConanException(f"Only core. values are allowed in --core-conf. Got {c}")
+        confs.loads("\n".join(core_confs))
+        confs.validate()
+        self._cli_core_confs = confs
+        # Last but not least, apply the new configuration
+        self.conan_api.reinit()
+
+    def reinit(self):
+        if self._new_config is not None:
+            self._new_config.clear()
+            self._populate_global_conf()

conan/api/subapi/local.py

@@ -118,3 +118,6 @@ def inspect(self, conanfile_path, remotes, lockfile, name=None, version=None, us
         conanfile = app.loader.load_named(conanfile_path, name=name, version=version, user=user,
                                           channel=channel, remotes=remotes, graph_lock=lockfile)
         return conanfile
+
+    def reinit(self):
+        self.editable_packages = EditablePackages(self._conan_api.home_folder)

conan/api/subapi/remotes.py

@@ -35,6 +35,9 @@ def __init__(self, conan_api):
         # Wraps an http_requester to inject proxies, certs, etc
         self._requester = ConanRequester(self.conan_api.config.global_conf, self.conan_api.cache_folder)
 
+    def reinit(self):
+        self._requester = ConanRequester(self.conan_api.config.global_conf, self.conan_api.cache_folder)
+
     def list(self, pattern=None, only_enabled=True):
         """
         Obtain a list of ``Remote`` objects matching the pattern.

conan/cli/command.py

@@ -4,7 +4,6 @@
 
 from conan.api.output import ConanOutput
 from conan.errors import ConanException
-from conan.internal.model.conf import CORE_CONF_PATTERN
 
 
 class OnceArgument(argparse.Action):
@@ -126,14 +125,7 @@ def parse_args(self, args=None, namespace=None):
             ConanOutput().error("The --lockfile-packages arg is private and shouldn't be used")
         global_conf = self._conan_api.config.global_conf
         if args.core_conf:
-            from conan.internal.model.conf import ConfDefinition
-            confs = ConfDefinition()
-            for c in args.core_conf:
-                if not CORE_CONF_PATTERN.match(c):
-                    raise ConanException(f"Only core. values are allowed in --core-conf. Got {c}")
-            confs.loads("\n".join(args.core_conf))
-            confs.validate()
-            global_conf.update_conf_definition(confs)
+            self._conan_api.config.set_core_confs(args.core_conf)
 
         # TODO: This might be even better moved to the ConanAPI so users without doing custom
         #  commands can benefit from it

conan/cli/commands/config.py

@@ -1,3 +1,4 @@
+from conan.api.input import UserInput
 from conan.api.model import Remote
 from conan.api.output import cli_out_write
 from conan.cli.command import conan_command, conan_subcommand, OnceArgument
@@ -132,3 +133,12 @@ def config_show(conan_api, parser, subparser, *args):
     args = parser.parse_args(*args)
 
     return conan_api.config.show(args.pattern)
+
+
+@conan_subcommand()
+def config_clean(conan_api, parser, subparser, *args):
+    """
+    Clean the configuration files in the Conan home folder. (Keeping installed packages)
+    """
+    parser.parse_args(*args)
+    conan_api.config.clean()

conan/internal/model/conf.py

@@ -706,3 +706,6 @@ def loads(self, text, profile=False):
     def validate(self):
         for conf in self._pattern_confs.values():
             conf.validate()
+
+    def clear(self):
+        self._pattern_confs.clear()

conans/client/migrations.py

@@ -3,7 +3,6 @@
 import textwrap
 
 from conan.api.output import ConanOutput
-from conan.api.subapi.config import ConfigAPI
 from conan.internal.default_settings import migrate_settings_file
 from conans.migrations import Migrator
 from conans.util.dates import timestamp_now
@@ -76,6 +75,7 @@ def migrate(home_folder):
 
 
 def _migrate_pkg_db_lru(cache_folder, old_version):
+    from conan.api.subapi.config import ConfigAPI
     config = ConfigAPI.load_config(cache_folder)
     storage = config.get("core.cache:storage_path") or os.path.join(cache_folder, "p")
     db_filename = os.path.join(storage, 'cache.sqlite3')

test/integration/command/config_test.py

@@ -2,7 +2,10 @@
 import os
 import textwrap
 
+import pytest
+
 from conan.api.conan_api import ConanAPI
+from conan.test.assets.genconanfile import GenConanfile
 from conan.internal.model.conf import BUILT_IN_CONFS
 from conan.test.utils.test_files import temp_folder
 from conan.test.utils.tools import TestClient
@@ -201,3 +204,67 @@ def test_config_show():
     tc.run("config show zlib/*:foo")
     assert "zlib/*:user.mycategory:foo" in tc.out
     assert "zlib/*:user.myothercategory:foo" in tc.out
+
+
+@pytest.mark.parametrize("storage_path", [None, "p", "../foo"])
+def test_config_clean(storage_path):
+    tc = TestClient(light=True)
+    absolut_storage_path = os.path.abspath(os.path.join(tc.current_folder, storage_path)) if storage_path else os.path.join(tc.cache_folder, "p")
+
+    storage = f"core.cache:storage_path={storage_path}" if storage_path else ""
+    tc.save_home({"global.conf": f"core.upload:retry=7\n{storage}",
+                  "extensions/compatibility/mycomp.py": "",
+                  "extensions/commands/cmd_foo.py": "",
+                  })
+
+    tc.run("profile detect --name=foo")
+    tc.run("remote add bar http://fakeurl")
+
+    tc.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+    tc.run("create .")
+
+    assert os.path.exists(absolut_storage_path)
+
+    tc.run("config clean")
+    tc.run("profile list")
+    assert "foo" not in tc.out
+    tc.run("remote list")
+    assert "bar" not in tc.out
+    tc.run("config show core.upload:retry")
+    assert "7" not in tc.out
+    assert not os.path.exists(os.path.join(tc.cache_folder, "extensions"))
+    assert os.path.exists(absolut_storage_path)
+
+
+def test_config_reinit():
+    custom_global_conf = "core.upload:retry=7"
+    global_conf_folder = temp_folder()
+    with open(os.path.join(global_conf_folder, "global.conf"), "w") as f:
+        f.write(custom_global_conf)
+
+    cache_folder = temp_folder()
+    conan_api = ConanAPI(cache_folder=cache_folder)
+    # Ensure reinitialization does not invalidate references
+    config_api = conan_api.config
+    assert config_api.global_conf.get("core.upload:retry", check_type=int) != 7
+
+    conan_api.config.install(global_conf_folder, verify_ssl=False)
+    # Already has an effect, the config installation reinitializes the config
+    assert config_api.global_conf.get("core.upload:retry", check_type=int) == 7
+
+
+def test_config_reinit_core_conf():
+    tc = TestClient(light=True)
+    tc.save_home({"extensions/commands/cmd_foo.py": textwrap.dedent("""
+        import json
+        from conan.cli.command import conan_command
+        from conan.api.output import ConanOutput
+
+        @conan_command()
+        def foo(conan_api, parser, *args, **kwargs):
+            ''' Foo '''
+            parser.parse_args(*args)
+            ConanOutput().info(f"Retry: {conan_api.config.global_conf.get('core.upload:retry', check_type=int)}")
+    """)})
+    tc.run("foo -cc core.upload:retry=7")
+    assert "Retry: 7" in tc.out