fix cache files and DB after forbidden downloads

conan/internal/cache/cache.py

@@ -170,6 +170,7 @@ def get_package_timestamp(self, pref):
 
     def remove_recipe(self, layout: RecipeLayout):
         layout.remove()
+        # FIXME: This is clearing package binaries from DB, but not from disk/layout
         self._db.remove_recipe(layout.reference)
 
     def remove_package(self, layout: RecipeLayout):

conans/client/remote_manager.py

@@ -52,19 +52,25 @@ def get_recipe(self, ref, remote):
         layout.export_remove()
 
         download_export = layout.download_export()
-        zipped_files = self._call_remote(remote, "get_recipe", ref, download_export)
-        remote_refs = self._call_remote(remote, "get_recipe_revisions_references", ref)
-        ref_time = remote_refs[0].timestamp
-        ref.timestamp = ref_time
-        # filter metadata files
-        # This could be also optimized in the download, avoiding downloading them, for performance
-        zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)}
-        # quick server package integrity check:
-        if "conanfile.py" not in zipped_files:
-            raise ConanException(f"Corrupted {ref} in '{remote.name}' remote: no conanfile.py")
-        if "conanmanifest.txt" not in zipped_files:
-            raise ConanException(f"Corrupted {ref} in '{remote.name}' remote: no conanmanifest.txt")
-        self._signer.verify(ref, download_export)
+        try:
+            zipped_files = self._call_remote(remote, "get_recipe", ref, download_export)
+            remote_refs = self._call_remote(remote, "get_recipe_revisions_references", ref)
+            ref_time = remote_refs[0].timestamp
+            ref.timestamp = ref_time
+            # filter metadata files
+            # This could be also optimized in download, avoiding downloading them, for performance
+            zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)}
+            # quick server package integrity check:
+            if "conanfile.py" not in zipped_files:
+                raise ConanException(f"Corrupted {ref} in '{remote.name}' remote: no conanfile.py")
+            if "conanmanifest.txt" not in zipped_files:
+                raise ConanException(f"Corrupted {ref} in '{remote.name}' remote: "
+                                     f"no conanmanifest.txt")
+            self._signer.verify(ref, download_export)
+        except BaseException:  # So KeyboardInterrupt also cleans things
+            ConanOutput(scope=str(ref)).error(f"Error downloading from remote '{remote.name}'")
+            self._cache.remove_recipe_layout(layout)
+            raise
         export_folder = layout.export()
         tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
 
@@ -126,7 +132,8 @@ def _get_package(self, layout, pref, remote, scoped_output):
             scoped_output.info("Downloaded package revision %s" % pref.revision)
         except NotFoundException:
             raise PackageNotFoundException(pref)
-        except BaseException as e:
+        except BaseException as e:  # So KeyboardInterrupt also cleans things
+            self._cache.remove_package_layout(layout)
             scoped_output.error("Exception while getting package: %s" % str(pref.package_id))
             scoped_output.error("Exception: %s %s" % (type(e), str(e)))
             raise

conans/client/rest/auth_manager.py

@@ -40,8 +40,8 @@ def call_rest_api_method(self, remote, method_name, *args, **kwargs):
         try:
             ret = getattr(rest_client, method_name)(*args, **kwargs)
             return ret
-        except ForbiddenException:
-            raise ForbiddenException("Permission denied for user: '%s'" % user)
+        except ForbiddenException as e:
+            raise ForbiddenException(f"Permission denied for user: '{user}': {e}")
         except AuthenticationException:
             # User valid but not enough permissions
             if user is None or token is None:

conans/client/rest/rest_client_v2.py

@@ -161,6 +161,8 @@ def _download_and_save_files(self, urls, dest_folder, files, parallel=False):
                                     verify_ssl=self.verify_ssl, retry=retry, retry_wait=retry_wait)
         for t in threads:
             t.join()
+        for t in threads:  # Need to join all before raising errors
+            t.raise_errors()
 
     def remove_all_packages(self, ref):
         """ Remove all packages from the specified reference"""

conans/test/integration/remote/broken_download_test.py

@@ -1,6 +1,9 @@
+import json
 import os
 
 import pytest
+from requests import Response
+
 from requests.exceptions import ConnectionError
 
 from conans.test.assets.genconanfile import GenConanfile
@@ -127,3 +130,77 @@ def DownloadFilesBrokenRequesterTimesTen(*args, **kwargs):
                                 "core.download:retry=11"}, path=client.cache.cache_folder)
     client.run("install --requires=lib/1.0@lasote/stable")
     assert 10 == str(client.out).count("Waiting 0 seconds to retry...")
+
+
+def test_forbidden_blocked_conanmanifest():
+    """ this is what happens when a server blocks downloading a specific file
+    """
+    server = TestServer()
+    servers = {"default": server}
+    client = TestClient(servers=servers, inputs=["admin", "password"])
+    client.save({"conanfile.py": GenConanfile()})
+    client.run("create . --name=lib --version=1.0")
+    client.run("upload lib/1.0* -c -r default")
+
+    class DownloadForbidden(TestRequester):
+        def get(self, url, **kwargs):
+            if "conanmanifest.txt" in url:
+                r = Response()
+                r._content = "Forbidden because of security!!!"
+                r.status_code = 403
+                return r
+            else:
+                return super(DownloadForbidden, self).get(url, **kwargs)
+
+    client = TestClient(servers=servers, inputs=["admin", "password"],
+                        requester_class=DownloadForbidden)
+    client.run("download lib/1.0 -r=default", assert_error=True)
+    assert "Forbidden because of security!!!" in client.out
+
+    client.run("list *")
+    assert "lib/1.0" not in client.out
+
+    client.run("install --requires=lib/1.0", assert_error=True)
+    assert "Forbidden because of security!!!" in client.out
+
+    client.run("list *")
+    assert "lib/1.0" not in client.out
+
+
+def test_forbidden_blocked_package_conanmanifest():
+    """ this is what happens when a server blocks downloading a specific file
+    """
+    server = TestServer()
+    servers = {"default": server}
+    client = TestClient(servers=servers, inputs=["admin", "password"])
+    client.save({"conanfile.py": GenConanfile()})
+    client.run("create . --name=lib --version=1.0")
+    client.run("upload lib/1.0* -c -r default")
+
+    class DownloadForbidden(TestRequester):
+        def get(self, url, **kwargs):
+            if "packages/" in url and "conanmanifest.txt" in url:
+                r = Response()
+                r._content = "Forbidden because of security!!!"
+                r.status_code = 403
+                return r
+            else:
+                return super(DownloadForbidden, self).get(url, **kwargs)
+
+    client = TestClient(servers=servers, inputs=["admin", "password"],
+                        requester_class=DownloadForbidden)
+    client.run("download lib/1.0 -r=default", assert_error=True)
+
+    def check_cache():
+        assert "Forbidden because of security!!!" in client.out
+        client.run("list *:* --format=json")
+        listjson = json.loads(client.stdout)
+        revisions = listjson["Local Cache"]["lib/1.0"]["revisions"]
+        packages = revisions["4d670581ccb765839f2239cc8dff8fbd"]["packages"]
+        assert packages == {}  # No binaries"
+
+    check_cache()
+
+    client.run("install --requires=lib/1.0", assert_error=True)
+    assert "Forbidden because of security!!!" in client.out
+    check_cache()

conans/util/thread.py

@@ -11,5 +11,7 @@ def run(self):
 
     def join(self, timeout=None):
         super().join(timeout=timeout)
+
+    def raise_errors(self):
         if self._exc:
             raise self._exc