osbuild: add new "build":"org.osbuild.containers-storage:..."

This commit adds support to construct a build root for a pipeline
from a container instead of a previous pipeline/tree. This is
based on the idea from Ondrej in issue 1804.

A bootc-image-builder manifest would look something like this:
```json
{
  "version": "2",
  "pipelines": [
    {
      "name": "image",
      "build": "org.osbuild.containers-storage:sha256:1234...",
...
   },
   ],
    "sources": {
        "org.osbuild.containers-storage": {
            "items": {
                f"sha256:1234...": {}
            }
        }
    }
}
```

Note that this is just an experiment and needs more thinking how
to abstract/generalize this. It is meant as a faster way to do the
org.osbuild.deploy-container stage does. We should also probably
enforce the uses the container hash instead of a tag when using
"build" for a start.

Closes: osbuild#1804

osbuild/formats/v2.py

@@ -391,7 +391,7 @@ def load(description: Dict, index: Index) -> Manifest:
     }
 
     for pipeline in pipelines:
-        if not pipeline.build:
+        if not pipeline.build or pipeline.build.startswith("org.osbuild.containers-storage:"):
             pipeline.runner = host_runner
             continue
 

osbuild/objectstore.py

@@ -318,6 +318,52 @@ def __fspath__(self) -> os.PathLike:
         return self.tree
 
 
+class ContainerMountTree:
+    """Access to a container based root filesystem.
+
+    An object that provides a similar interface to
+    `objectstore.Object` and provides access to a container
+    """
+
+    def __init__(self, from_container: str) -> None:
+        self._from_container = None
+        self._root = None
+        self.init(from_container)
+
+    def init(self, from_container: str) -> None:
+        self._from_container = from_container
+        result = subprocess.run(
+            ["podman", "image", "mount", from_container],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            encoding="utf-8",
+            check=False,
+        )
+        if result.returncode != 0:
+            code = result.returncode
+            msg = result.stderr.strip()
+            raise RuntimeError(f"Failed to mount image ({code}): {msg}")
+        self._root = result.stdout.strip()
+
+    @property
+    def tree(self) -> os.PathLike:
+        if not self._root:
+            raise AssertionError(f"ContainerMountTree for {self._from_container} not initialized")
+        return self._root
+
+    def cleanup(self):
+        if self._root:
+            subprocess.run(
+                ["podman", "image", "umount", self._from_container],
+                stdout=subprocess.DEVNULL,
+                check=True,
+            )
+            self._root = None
+
+    def __fspath__(self) -> os.PathLike:
+        return self.tree
+
+
 class ObjectStore(contextlib.AbstractContextManager):
     def __init__(self, store: PathLike):
         self.cache = FsCache("osbuild", store)

osbuild/pipeline.py

@@ -11,7 +11,7 @@
 from .devices import Device, DeviceManager
 from .inputs import Input, InputManager
 from .mounts import Mount, MountManager
-from .objectstore import ObjectStore
+from .objectstore import ContainerMountTree, ObjectStore
 from .sources import Source
 from .util import osrelease
 
@@ -313,6 +313,9 @@ def build_stages(self, object_store, monitor, libdir, debug_break="", stage_time
 
         if not self.build:
             build_tree = object_store.host_tree
+        elif self.build.startswith("org.osbuild.containers-storage:"):
+            cnt_id = self.build.removeprefix("org.osbuild.containers-storage:")
+            build_tree = ContainerMountTree(cnt_id)
         else:
             build_tree = object_store.get(self.build)
 
@@ -365,6 +368,10 @@ def build_stages(self, object_store, monitor, libdir, debug_break="", stage_time
             if stage.checkpoint:
                 object_store.commit(tree, stage.id)
 
+        # XXX: needs a test but pretty sure we need this (i.e. this is
+        # a pre-existing leak) as AFAICT HostTree is never umounted
+        # otherwise (and ContainerMountTree now of course)
+        build_tree.cleanup()
         tree.finalize()
 
         return results
@@ -457,7 +464,7 @@ def depsolve(self, store: ObjectStore, targets: Iterable[str]) -> List[str]:
 
             # Add all dependencies to the stack of things to check,
             # starting with the build pipeline, if there is one
-            if pl.build:
+            if pl.build and not pl.build.startswith("org.osbuild.containers-storage:"):
                 check.append(self.get(pl.build))
 
             # Stages depend on other pipeline via pipeline inputs.

test/run/test_buildroot.py

@@ -0,0 +1,47 @@
+import contextlib
+import json
+import os
+import pathlib
+import shutil
+import subprocess
+
+import pytest
+
+from osbuild.testutil import make_container
+from .. import test
+from .test_exports import osbuild_fixture, testing_libdir_fixture
+
+
+@pytest.mark.skipif(os.getuid() != 0, reason="root-only")
+def test_build_root_from_container_registry(osb, tmp_path, testing_libdir):
+    cnt_ref = "registry.access.redhat.com/ubi9:latest"
+    with make_container(tmp_path, {"/usr/bin/buildroot-from-container": "foo"}, cnt_ref) as fake_cnt_tag:
+        img_id = subprocess.check_output(["podman", "inspect", "--format={{.Id}}", fake_cnt_tag], text=True).strip()
+        jsondata = json.dumps({
+            "version": "2",
+            "pipelines": [
+                {
+                    "name": "image",
+                    "build": f"org.osbuild.containers-storage:sha256:{img_id}",
+                    "stages": [
+                        {
+                            "type": "org.osbuild.testing.injectpy",
+                            "options": {
+                                "code": [
+                                    'import os.path',
+                                    'assert os.path.exists("/usr/bin/buildroot-from-container")',
+                                ],
+                            },
+                        },
+                    ],
+                },
+            ],
+            "sources": {
+                "org.osbuild.containers-storage": {
+                    "items": {
+                        f"sha256:{img_id}": {}
+                    }
+                }
+            }
+        })
+        osb.compile(jsondata, output_dir=tmp_path, exports=["image"], libdir=testing_libdir)

test/run/test_exports.py

@@ -58,7 +58,7 @@ def testing_libdir_fixture(tmpdir_factory):
     # in buildroot.py
     (fake_libdir_path / "osbuild").mkdir()
     # construct minimal viable libdir from current checkout
-    for d in ["stages", "runners", "schemas", "assemblers"]:
+    for d in ["stages", "runners", "schemas", "assemblers", "sources"]:
         subprocess.run(
             ["cp", "-a", os.fspath(project_path / d), f"{fake_libdir_path}"],
             check=True)