Implement a check for every storage whether a repository is available

backend/copr_backend/background_worker_build.py

@@ -348,10 +348,11 @@ def _wait_for_repo(self):
             # we don't need copr_base repodata for srpm builds
             return
 
-        repodata = os.path.join(self.job.chroot_dir, "repodata/repomd.xml")
         waiting_since = time.time()
         while time.time() - waiting_since < 60:
-            if os.path.exists(repodata):
+            exists = self.storage.repository_exists(
+                self.job.project_dirname, self.job.chroot)
+            if exists:
                 return
 
             # Either (a) the very first copr-repo run in this chroot dir

backend/copr_backend/storage.py

@@ -5,6 +5,8 @@
 import os
 import json
 import shutil
+from urllib.parse import urlparse
+import requests
 from copr_common.enums import StorageEnum
 from copr_backend.helpers import call_copr_repo, build_chroot_log_name
 from copr_backend.pulp import PulpClient
@@ -85,6 +87,12 @@ def delete_builds(self, dirname, chroot_builddirs, build_ids):
         """
         raise NotImplementedError
 
+    def repository_exists(self, dirname, chroot):
+        """
+        Does a repository exist?
+        """
+        raise NotImplementedError
+
 
 class BackendStorage(Storage):
     """
@@ -172,6 +180,11 @@ def delete_builds(self, dirname, chroot_builddirs, build_ids):
                         self.log.debug("can't remove %s", log_path)
         return result
 
+    def repository_exists(self, dirname, chroot):
+        repodata = os.path.join(self.opts.destdir, self.owner, dirname,
+                                chroot, "repodata", "repomd.xml")
+        return os.path.exists(repodata)
+
 
 class PulpStorage(Storage):
     """
@@ -182,7 +195,7 @@ def __init__(self, *args, **kwargs):
         self.client = PulpClient.create_from_config_file()
 
     def init_project(self, dirname, chroot):
-        repository = self._repository_name(chroot)
+        repository = self._repository_name(chroot, dirname)
         response = self.client.create_repository(repository)
         if not response.ok and "This field must be unique" not in response.text:
             self.log.error("Failed to create a Pulp repository %s because of %s",
@@ -193,7 +206,7 @@ def init_project(self, dirname, chroot):
         # mentioned by its href, not name
         repository = self._get_repository(chroot)
 
-        distribution = self._distribution_name(chroot)
+        distribution = self._distribution_name(chroot, dirname)
         response = self.client.create_distribution(distribution, repository)
         if not response.ok and "This field must be unique" not in response.text:
             self.log.error("Failed to create a Pulp distribution %s because of %s",
@@ -342,15 +355,38 @@ def delete_builds(self, dirname, chroot_builddirs, build_ids):
 
         return result
 
+    def repository_exists(self, dirname, chroot):
+        name = self._distribution_name(chroot, dirname)
+        response = self.client.get_distribution(name)
+        if not response.ok:
+            return False
+
+        data = response.json()
+        if data["count"] == 0:
+            return False
+
+        distribution = response.json()["results"][0]
+
+        # For some instances (local container) the distribution base_url
+        # contains only path, for some instances (hosted STG) it returns fully
+        # qualified URL. The problem is that there is a lot of magic in the
+        # hosted deployment in order to provide the data publicly without
+        # a Red Hat login. And we cannot use the returned URL, only its path.
+        path = urlparse(distribution["base_url"]).path.lstrip("/")
+        host = self.client.config["base_url"].rstrip("/")
+        repodata = "{0}/{1}/repodata/repomd.xml".format(host, path)
+        response = requests.head(repodata)
+        return response.ok
+
     def _repository_name(self, chroot, dirname=None):
         return "/".join([
             self.owner,
             dirname or self.project,
             chroot,
         ])
 
-    def _distribution_name(self, chroot):
-        repository = self._repository_name(chroot)
+    def _distribution_name(self, chroot, dirname=None):
+        repository = self._repository_name(chroot, dirname)
         if self.devel:
             return "{0}-devel".format(repository)
         return repository