use neww uuids for new builds folders in cache (#13833)

* use neww uuids for new builds folders in cache

* fixes

* fix test

* wip

* fixes

* fixes

conan/api/subapi/cache.py

@@ -1,4 +1,4 @@
-import shutil
+import os
 
 from conan.internal.conan_app import ConanApp
 from conan.internal.integrity_check import IntegrityChecker
@@ -64,6 +64,16 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True)
         app = ConanApp(self.conan_api.cache_folder)
         if temp:
             rmdir(app.cache.temp_folder)
+            # Clean those build folders that didn't succeed to create a package and wont be in DB
+            builds_folder = app.cache.builds_folder
+            if os.path.isdir(builds_folder):
+                for subdir in os.listdir(builds_folder):
+                    folder = os.path.join(builds_folder, subdir)
+                    manifest = os.path.join(folder, "p", "conanmanifest.txt")
+                    info = os.path.join(folder, "p", "conaninfo.txt")
+                    if not os.path.exists(manifest) or not os.path.exists(info):
+                        rmdir(folder)
+
         for ref, ref_bundle in package_list.refs():
             ref_layout = app.cache.ref_layout(ref)
             if source:

conan/api/subapi/remove.py

@@ -33,11 +33,8 @@ def all_recipe_packages(self, ref: RecipeReference, remote: Remote = None):
     @staticmethod
     def _remove_all_local_packages(app, ref):
         # Get all the prefs and all the prevs
-        pkg_ids = app.cache.get_package_references(ref)
-        all_package_revisions = []
-        for pkg_id in pkg_ids:
-            all_package_revisions.extend(app.cache.get_package_revisions_references(pkg_id))
-        for pref in all_package_revisions:
+        pkg_ids = app.cache.get_package_references(ref, only_latest_prev=False)
+        for pref in pkg_ids:
             package_layout = app.cache.pkg_layout(pref)
             app.cache.remove_package_layout(package_layout)
 

conan/internal/cache/cache.py

@@ -1,5 +1,6 @@
 import hashlib
 import os
+import uuid
 
 from conan.internal.cache.conan_reference_layout import RecipeLayout, PackageLayout
 # TODO: Random folders are no longer accessible, how to get rid of them asap?
@@ -56,8 +57,11 @@ def _get_tmp_path(ref: RecipeReference):
     @staticmethod
     def _get_tmp_path_pref(pref):
         # The reference will not have revision, but it will be always constant
-        h = pref.ref.name[:5] + DataCache._short_hash_path(pref.repr_notime())
-        return os.path.join("t", h)
+        assert pref.revision is None
+        assert pref.timestamp is None
+        random_id = str(uuid.uuid4())
+        h = pref.ref.name[:5] + DataCache._short_hash_path(pref.repr_notime() + random_id)
+        return os.path.join("b", h)
 
     @staticmethod
     def _get_path(ref: RecipeReference):
@@ -173,31 +177,25 @@ def remove_recipe(self, layout: RecipeLayout):
         # FIXME: This is clearing package binaries from DB, but not from disk/layout
         self._db.remove_recipe(layout.reference)
 
-    def remove_package(self, layout: RecipeLayout):
+    def remove_package(self, layout: PackageLayout):
         layout.remove()
         self._db.remove_package(layout.reference)
 
     def assign_prev(self, layout: PackageLayout):
         pref = layout.reference
 
-        new_path = self._get_path_pref(pref)
-
-        full_path = self._full_path(new_path)
-        rmdir(full_path)
-
-        renamedir(self._full_path(layout.base_folder), full_path)
-        layout._base_folder = os.path.join(self.base_folder, new_path)
-
         build_id = layout.build_id
         pref.timestamp = revision_timestamp_now()
         # Wait until it finish to really update the DB
+        relpath = os.path.relpath(layout.base_folder, self.base_folder)
         try:
-            self._db.create_package(new_path, pref, build_id)
+            self._db.create_package(relpath, pref, build_id)
         except ConanReferenceAlreadyExistsInDB:
+            # TODO: Optimize this into 1 single UPSERT operation
             # This was exported before, making it latest again, update timestamp
-            self._db.update_package_timestamp(pref)
-
-        return new_path
+            pkg_layout = self.get_package_layout(pref)
+            pkg_layout.remove()
+            self._db.update_package_timestamp(pref, path=relpath)
 
     def assign_rrev(self, layout: RecipeLayout):
         """ called at export, once the exported recipe revision has been computed, it

conan/internal/cache/db/cache_database.py

@@ -43,8 +43,8 @@ def get_latest_package_reference(self, ref):
     def update_recipe_timestamp(self, ref):
         self._recipes.update_timestamp(ref)
 
-    def update_package_timestamp(self, pref: PkgReference):
-        self._packages.update_timestamp(pref)
+    def update_package_timestamp(self, pref: PkgReference, path: str):
+        self._packages.update_timestamp(pref, path=path)
 
     def remove_recipe(self, ref: RecipeReference):
         # Removing the recipe must remove all the package binaries too from DB

conan/internal/cache/db/packages_table.py

@@ -82,11 +82,11 @@ def create(self, path, pref: PkgReference, build_id):
             except sqlite3.IntegrityError:
                 raise ConanReferenceAlreadyExistsInDB(f"Reference '{repr(pref)}' already exists")
 
-    def update_timestamp(self, pref: PkgReference):
+    def update_timestamp(self, pref: PkgReference, path: str):
         assert pref.revision
         assert pref.timestamp
         where_clause = self._where_clause(pref)
-        set_clause = self._set_clause(pref)
+        set_clause = self._set_clause(pref, path=path)
         query = f"UPDATE {self.table_name} " \
                 f"SET {set_clause} " \
                 f"WHERE {where_clause};"

conans/client/cache/cache.py

@@ -58,6 +58,10 @@ def temp_folder(self):
         # TODO: Improve the path definitions, this is very hardcoded
         return os.path.join(self.cache_folder, "p", "t")
 
+    @property
+    def builds_folder(self):
+        return os.path.join(self.cache_folder, "p", "b")
+
     def create_export_recipe_layout(self, ref: RecipeReference):
         return self._data_cache.create_export_recipe_layout(ref)
 

conans/test/functional/layout/test_local_commands.py

@@ -222,6 +222,8 @@ def package(self):
 
     # Doing a conan create: Same as export-pkg, there is "my_package" in the cache
     client.run("create . ")
+    pref = client.get_latest_package_reference(ref)
+    pf = client.get_latest_pkg_layout(pref).package()
     assert "my_package" not in pf
     assert os.path.exists(os.path.join(pf, "downloaded.h"))
     assert os.path.exists(os.path.join(pf, "generated.h"))

conans/test/integration/command/upload/upload_test.py

@@ -74,6 +74,8 @@ def test_upload_force(self):
         if platform.system() == "Linux":
             client.run("remove '*' -c")
             client.create(ref, conanfile=GenConanfile().with_package_file("myfile.sh", "foo"))
+            package_folder = client.get_latest_pkg_layout(pref).package()
+            package_file_path = os.path.join(package_folder, "myfile.sh")
             os.system('chmod +x "{}"'.format(package_file_path))
             self.assertTrue(os.stat(package_file_path).st_mode & stat.S_IXUSR)
             client.run("upload * --confirm -r default")
@@ -90,6 +92,8 @@ def test_upload_force(self):
         if platform.system() == "Linux":
             client.run("remove '*' -c")
             client.run("install --requires={}".format(ref))
+            package_folder = client.get_latest_pkg_layout(pref).package()
+            package_file_path = os.path.join(package_folder, "myfile.sh")
             # Owner with execute permissions
             self.assertTrue(os.stat(package_file_path).st_mode & stat.S_IXUSR)
 

conans/test/integration/command_v2/test_cache_clean.py

@@ -31,15 +31,21 @@ def test_cache_clean():
 
 def test_cache_clean_all():
     c = TestClient()
-    c.save({"pkg1/conanfile.py": GenConanfile("pkg", "0.1").with_package("error"),
-            "pkg2/conanfile.py": GenConanfile("pkg", "0.2")})
+    c.save({"pkg1/conanfile.py": GenConanfile("pkg", "0.1").with_class_attribute("revision_mode='scm'"),
+            "pkg2/conanfile.py": GenConanfile("pkg", "0.2").with_package("error"),
+            "pkg3/conanfile.py": GenConanfile("pkg", "0.3")})
     c.run("create pkg1", assert_error=True)
-    c.run("create pkg2")
-    pref = c.created_package_reference("pkg/0.2")
-    folder = os.path.join(c.cache_folder, "p", "t")
-    assert len(os.listdir(folder)) == 1
+    c.run("create pkg2", assert_error=True)
+    c.run("create pkg3")
+    pref = c.created_package_reference("pkg/0.3")
+    temp_folder = os.path.join(c.cache_folder, "p", "t")
+    assert len(os.listdir(temp_folder)) == 1  # Failed export was here
+    builds_folder = os.path.join(c.cache_folder, "p", "b")
+    assert len(os.listdir(builds_folder)) == 2  # both builds are here
     c.run('cache clean')
-    assert not os.path.exists(folder)
+    assert not os.path.exists(temp_folder)
+    assert len(os.listdir(builds_folder)) == 1  # only correct pkg/0.3 remains
+    # Check correct package removed all
     ref_layout = c.get_latest_ref_layout(pref.ref)
     pkg_layout = c.get_latest_pkg_layout(pref)
     assert not os.path.exists(ref_layout.source())
@@ -50,4 +56,61 @@ def test_cache_clean_all():
     # A second clean like this used to crash
     # as it tried to delete a folder that was not there and tripped shutils up
     c.run('cache clean')
-    assert not os.path.exists(folder)
+    assert not os.path.exists(temp_folder)
+
+
+def test_cache_multiple_builds_same_prev_clean():
+    """
+    Different consecutive builds will create different folders, even if for the
+    same exact prev, leaving trailing non-referenced folders
+    """
+    c = TestClient()
+    c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+    c.run("create .")
+    create_out = c.out
+    c.run("cache path pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709")
+    path1 = str(c.stdout)
+    assert path1 in create_out
+    c.run("create .")
+    create_out = c.out
+    c.run("cache path pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709")
+    path2 = str(c.stdout)
+    assert path2 in create_out
+    assert path1 != path2
+
+    builds_folder = os.path.join(c.cache_folder, "p", "b")
+    assert len(os.listdir(builds_folder)) == 1  # only one build
+    c.run('cache clean')
+    assert len(os.listdir(builds_folder)) == 1  # one build not cleaned
+    c.run('remove * -c')
+    assert len(os.listdir(builds_folder)) == 0  # no folder remain
+
+
+def test_cache_multiple_builds_diff_prev_clean():
+    """
+    Different consecutive builds will create different folders, even if for the
+    same exact prev, leaving trailing non-referenced folders
+    """
+    c = TestClient()
+    package_lines = 'save(self, os.path.join(self.package_folder, "foo.txt"), str(time.time()))'
+    gen = GenConanfile("pkg", "0.1").with_package(package_lines).with_import("import os, time") \
+                                    .with_import("from conan.tools.files import save")
+    c.save({"conanfile.py": gen})
+    c.run("create .")
+    create_out = c.out
+    c.run("cache path pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709")
+    path1 = str(c.stdout)
+    assert path1 in create_out
+    c.run("create .")
+    create_out = c.out
+    c.run("cache path pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709")
+    path2 = str(c.stdout)
+    assert path2 in create_out
+    assert path1 != path2
+
+    builds_folder = os.path.join(c.cache_folder, "p", "b")
+    assert len(os.listdir(builds_folder)) == 2  # both builds are here
+    c.run('cache clean')
+    assert len(os.listdir(builds_folder)) == 2  # two builds will remain, both are valid
+    c.run('remove * -c')
+    assert len(os.listdir(builds_folder)) == 0  # no folder remain

conans/test/integration/test_package_python_files.py

@@ -47,6 +47,10 @@ def package(self):
     client.run("remove * -c")
     client.run("download pkg/0.1#*:* -r default")
 
+    # The download will be in a different pkg folder now.
+    pref = client.get_latest_package_reference(ref, NO_SETTINGS_PACKAGE_ID)
+    pkg_folder = client.get_latest_pkg_layout(pref).package()
+
     assert os.path.isfile(os.path.join(export_sources, "myfile.pyc"))
     assert os.path.isfile(os.path.join(export_sources, "myfile.pyo"))
     assert not os.path.isfile(os.path.join(export_sources, ".DS_Store"))