queue: fix missing cycles with build-only deps of deps

We only looked at the dependencies of a package that are needed for building,
but for detecting build cycles we also have to look at all transitive deps.

Unless the dependency is already finished, then we can ignore its build deps,
even if they are not finished yet.

The test shows such a case where things indirectly create a cycle via cmake.

Fixes msys2#91

msys2_autobuild/queue.py

@@ -137,12 +137,16 @@ def get_rdepends(self, build_type: BuildType) -> "Dict[ArchType, Set[Package]]":
 
 
 def get_buildqueue() -> List[Package]:
-    pkgs = []
     session = get_requests_session()
     r = session.get("https://packages.msys2.org/api/buildqueue2", timeout=REQUESTS_TIMEOUT)
     r.raise_for_status()
 
-    for received in r.json():
+    return parse_buildqueue(r.text)
+
+
+def parse_buildqueue(payload: str) -> List[Package]:
+    pkgs = []
+    for received in json.loads(payload):
         pkg = Package(received)
         pkg['repo'] = pkg['repo_url'].split('/')[-1]
         pkgs.append(pkg)
@@ -188,23 +192,46 @@ def pkg_is_finished(pkg: Package, build_type: BuildType) -> bool:
             PackageStatus.FINISHED_BUT_INCOMPLETE,
         ]
 
+    # Transitive dependencies of a package. Excluding branches where a root is finished
+    def get_buildqueue_deps(pkg: Package, build_type: ArchType) -> "Dict[ArchType, Set[Package]]":
+        start = (build_type, pkg)
+        todo = set([start])
+        done = set()
+        result = set()
+
+        while todo:
+            build_type, pkg = todo.pop()
+            item = (build_type, pkg)
+            done.add(item)
+            if pkg_is_finished(pkg, build_type):
+                continue
+            result.add(item)
+            for dep_build_type, deps in pkg.get_depends(build_type).items():
+                for dep in deps:
+                    dep_item = (dep_build_type, dep)
+                    if dep_item not in done:
+                        todo.add(dep_item)
+        result.discard(start)
+
+        d: Dict[ArchType, Set[Package]] = {}
+        for build_type, pkg in result:
+            d.setdefault(build_type, set()).add(pkg)
+        return d
+
     for pkg in pkgs:
         for build_type in pkg.get_build_types():
             if build_type_is_src(build_type):
                 continue
-            if pkg_is_finished(pkg, build_type):
-                continue
             build_type = cast(ArchType, build_type)
-            for dep_type, deps in pkg.get_depends(build_type).items():
+            for dep_build_type, deps in get_buildqueue_deps(pkg, build_type).items():
                 for dep in deps:
-                    if pkg_is_finished(dep, dep_type):
-                        continue
                     # manually broken cycle
-                    if pkg.is_optional_dep(dep, dep_type) or dep.is_optional_dep(pkg, build_type):
+                    if pkg.is_optional_dep(dep, dep_build_type) or dep.is_optional_dep(pkg, build_type):
                         continue
-                    dep_deps = dep.get_depends(dep_type)
+                    dep_deps = get_buildqueue_deps(dep, dep_build_type)
                     if pkg in dep_deps.get(build_type, set()):
                         cycles.add(tuple(sorted([pkg, dep], key=lambda p: p["name"])))  # type: ignore
+
     return cycles
 
 

tests/main_test.py

@@ -1,7 +1,90 @@
 # type: ignore
 
 from msys2_autobuild.utils import parse_optional_deps
+from msys2_autobuild.queue import parse_buildqueue, get_cycles
 
 
 def test_parse_optional_deps():
     assert parse_optional_deps("a:b,c:d,a:x") == {'a': ['b', 'x'], 'c': ['d']}
+
+
+def test_get_cycles():
+    buildqueue = """
+[
+  {
+    "name": "c-ares",
+    "version": "1.34.2-1",
+    "version_repo": "1.33.1-1",
+    "repo_url": "https://github.com/msys2/MSYS2-packages",
+    "repo_path": "c-ares",
+    "source": true,
+    "builds": {
+      "msys": {
+        "packages": [
+          "libcares",
+          "libcares-devel"
+        ],
+        "depends": {
+          "msys": [
+            "libnghttp2",
+            "libuv"
+          ]
+        },
+        "new": false
+      }
+    }
+  },
+  {
+    "name": "nghttp2",
+    "version": "1.64.0-1",
+    "version_repo": "1.63.0-1",
+    "repo_url": "https://github.com/msys2/MSYS2-packages",
+    "repo_path": "nghttp2",
+    "source": true,
+    "builds": {
+      "msys": {
+        "packages": [
+          "libnghttp2",
+          "libnghttp2-devel",
+          "nghttp2"
+        ],
+        "depends": {
+          "msys": [
+            "libcares",
+            "libcares-devel"
+          ]
+        },
+        "new": false
+      }
+    }
+  },
+  {
+    "name": "libuv",
+    "version": "1.49.2-1",
+    "version_repo": "1.49.1-1",
+    "repo_url": "https://github.com/msys2/MSYS2-packages",
+    "repo_path": "libuv",
+    "source": true,
+    "builds": {
+      "msys": {
+        "packages": [
+          "libuv",
+          "libuv-devel"
+        ],
+        "depends": {
+          "msys": [
+            "libnghttp2"
+          ]
+        },
+        "new": false
+      }
+    }
+  }
+]"""
+
+    pkgs = parse_buildqueue(buildqueue)
+    cycles = get_cycles(pkgs)
+    assert len(cycles) == 3
+    assert (pkgs[0], pkgs[2]) in cycles
+    assert (pkgs[0], pkgs[1]) in cycles
+    assert (pkgs[2], pkgs[1]) in cycles