Forbid failing incidents from being scheduled in aggregates

openqabot/types/aggregate.py

@@ -95,8 +95,16 @@ def __call__(
                     # if filtering embargoed updates is off we ignoring this field
                     else:
                         valid_incidents.append(i)
+
+            # filter out incidents which have already failed in single incident tests
+            filtered_incidents = [
+                incident
+                for incident in valid_incidents
+                if not incident.has_failures(token)
+            ]
+
             for issue, template in self.test_issues.items():
-                for inc in valid_incidents:
+                for inc in filtered_incidents:
                     if (
                         Repos(template.product, template.version, issues_arch)
                         in inc.channels

openqabot/types/incident.py

@@ -3,11 +3,15 @@
 import re
 from logging import getLogger
 from typing import Dict, List, Tuple
+from urllib import request
 
+# Make sure we have the right dashboard URL
+from .. import QEM_DASHBOARD
 from . import ArchVer, Repos
-from ..errors import EmptyChannels, EmptyPackagesError, NoRepoFoundError
+from ..errors import EmptyChannels, EmptyPackagesError, NoRepoFoundError, NoResultsError
 from ..loader.repohash import get_max_revision
 
+
 log = getLogger("bot.types.incident")
 version_pattern = re.compile(r"(\d+(?:[.-](?:SP)?\d+)?)")
 
@@ -144,3 +148,30 @@ def contains_package(self, requires: List[str]) -> bool:
                 if package.startswith(req) and package != "kernel-livepatch-tools":
                     return True
         return False
+
+    def has_failures(self, token) -> bool:
+        results = request.get(
+            QEM_DASHBOARD + "/api/jobs/incident/" + str(self.id), headers=token
+        ).json()
+
+        # This should be wrapped by a verbose flag so we return faster
+        for res in results:
+            if res["status"] == "passed":
+                continue
+            else:
+                log.info(
+                    "Found failed, not-ignored job /t%s for incident %s",
+                    res["job_id"],
+                    self.id,
+                )
+                break
+
+        if not results:
+            raise NoResultsError(
+                "Job setting %s not found for incident" % (str(self.id))
+            )
+
+        # Return true if there are any failed jobs, so if results is not empty
+        # and there is at least one failed job, return true, any() returns false
+        # if the iterable is empty
+        return any(r["status"] == "failed" for r in results)

tests/test_aggregate.py

@@ -91,15 +91,24 @@ class Repos(NamedTuple):
         arch: str
 
     class MockIncident:
-        def __init__(self, repo, embargoed):
+        def __init__(self, repo, embargoed, has_failures, mocked_incident=42):
+            self.id = mocked_incident
             self.livepatch = None
             self.staging = None
             self.channels = [repo]
             self.embargoed = embargoed
+            self._has_failures = has_failures
 
-    def _func(product, version, arch, embargoed=False):
+        def has_failures(self, token):
+            return self._has_failures
+
+    def _func(
+        product, version, arch, embargoed=False, has_failures=False, mocked_incident=42
+    ):
         repo = Repos(product=product, version=version, arch=arch)
-        return MockIncident(repo, embargoed=embargoed)
+        return MockIncident(
+            repo, embargoed=embargoed, has_failures=False, mocked_incident=42
+        )
 
     return _func
 
@@ -114,12 +123,37 @@ def test_aggregate_call_with_test_issues(request_mock, incident_mock, monkeypatc
     my_config["test_issues"] = {"AAAAAAA": "BBBBBBBBB:CCCCCCCC"}
     acc = Aggregate("", settings={}, config=my_config)
     res = acc(
-        incidents=[incident_mock(product="BBBBBBBBB", version="CCCCCCCC", arch="ciao")],
+        incidents=[
+            incident_mock(
+                product="BBBBBBBBB", version="CCCCCCCC", arch="ciao", has_failures=False
+            )
+        ],
         token=None,
         ci_url=None,
     )
     assert len(res) == 1
 
+    res.append(
+        incident_mock(
+            product="BBBBBBBBB",
+            version="CCCCCCCC",
+            arch="the_heaven_incident",
+            has_failures=False,
+        )
+    )
+    assert len(res) == 2
+
+    res.append(
+        incident_mock(
+            product="BBBBBBBBB",
+            version="CCCCCCCC",
+            arch="the_demon_incident",
+            has_failures=True,
+            mocked_incident=666,
+        )
+    )
+    assert len(res) == 2  # the demon incident is not added, because it has failures
+
 
 def test_aggregate_call_pc_pint(request_mock, monkeypatch):
     """