Add a test for building OCI images

closes #461

CHANGES/461.bugfix

@@ -0,0 +1 @@
+Fixed the machinery for building OCI images.

docs/installation.rst

@@ -69,16 +69,9 @@ Run Services
    sudo systemctl restart pulpcore-worker@1
    sudo systemctl restart pulpcore-worker@2
 
-Enable OCI Container Image building
------------------------------------
-
-Pulp container plugin can be used to build an OCI format image from a Containerfile. The plugin uses
-`buildah <https://github.com/containers/buildah/>`_ to build the container image. Buildah 1.14+
-must be installed on the same machine that is running pulpcore-worker processes.
-
-The pulpcore-worker processes needs to have `/usr/bin/` in its `PATH`. The user that is running
-pulpcore-worker process needs to be able to manage subordinate user ids and group ids. The range of
-subordinate user ids is specified in `/etc/subuid` and the range of subordinate group ids is
-specified in `/etc/subgid`. More details can be found in `buildah documentation <https://github.com
-/containers/libpod/blob/master/docs/tutorials/rootless_tutorial.md#enable-user-namespaces-on-rhel7-
-machines>`_.
+OCI Container Image building
+----------------------------
+
+The plugin can be used to build an OCI format image from a Containerfile. The plugin uses podman
+to build containers. Refer to `podman-build documentation <https://docs.podman.io/en/latest/markdown/podman-build.1.html>`_
+for more details.

docs/workflows/build-containerfile.rst

@@ -8,8 +8,6 @@ Build an OCI image from a Containerfile
     All container build APIs are tech preview in Pulp Container 2.1. Backwards compatibility when
     upgrading is not guaranteed.
 
-    `buildah` needs to be installed to enable image building.
-
 Users can add new images to a container repository by uploading a Containerfile. The syntax for
 Containerfile is the same as for a Dockerfile. The same REST API endpoint also accepts a JSON
 string that maps artifacts in Pulp to a filename. Any artifacts passed in are available inside the

pulp_container/app/serializers.py

@@ -496,27 +496,21 @@ class CopySerializer(ValidateFieldsMixin, serializers.Serializer):
     )
 
     def validate(self, data):
-        """Ensure that source_repository or source_rpository_version is pass, but not both."""
+        """Ensure that source_repository or source_repository_version is passed, but not both."""
         data = super().validate(data)
 
         repository = data.pop("source_repository", None)
         repository_version = data.get("source_repository_version")
         if not repository and not repository_version:
             raise serializers.ValidationError(
-                _("Either the 'repository' or 'repository_version' need to be specified")
+                _("Either the 'repository' or 'repository_version' needs to be specified")
             )
         elif not repository and repository_version:
             return data
         elif repository and not repository_version:
-            version = repository.latest_version()
-            if version:
-                new_data = {"source_repository_version": version}
-                new_data.update(data)
-                return new_data
-            else:
-                raise serializers.ValidationError(
-                    detail=_("Source repository has no version available to copy content from")
-                )
+            new_data = {"source_repository_version": repository.latest_version()}
+            new_data.update(data)
+            return new_data
         raise serializers.ValidationError(
             _(
                 "Either the 'repository' or 'repository_version' need to be specified "
@@ -640,10 +634,12 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer):
         lookup_field="pk",
         view_name="artifacts-detail",
         queryset=Artifact.objects.all(),
-        help_text=_("Artifact representing the Containerfile that should be used to run buildah."),
+        help_text=_(
+            "Artifact representing the Containerfile that should be used to run podman-build."
+        ),
     )
     containerfile = serializers.FileField(
-        help_text=_("An uploaded Containerfile that should be used to run buildah."),
+        help_text=_("An uploaded Containerfile that should be used to run podman-build."),
         required=False,
     )
     tag = serializers.CharField(
@@ -693,7 +689,7 @@ def validate(self, data):
                 try:
                     artifact = artifactfield.run_validation(data=url)
                     artifact.touch()
-                    artifacts[artifact.pk] = relative_path
+                    artifacts[str(artifact.pk)] = relative_path
                 except serializers.ValidationError as e:
                     # Append the URL of missing Artifact to the error message
                     e.detail[0] = "%s %s" % (e.detail[0], url)

pulp_container/app/tasks/builder.py

@@ -13,7 +13,7 @@
     Tag,
 )
 from pulp_container.constants import MEDIA_TYPE
-from pulpcore.plugin.models import Artifact, ContentArtifact
+from pulpcore.plugin.models import Artifact, ContentArtifact, Content
 
 
 def get_or_create_blob(layer_json, manifest, path):
@@ -33,7 +33,7 @@ def get_or_create_blob(layer_json, manifest, path):
         blob = Blob.objects.get(digest=layer_json["digest"])
         blob.touch()
     except Blob.DoesNotExist:
-        layer_file_name = "{}{}".format(path, layer_json["digest"][7:])
+        layer_file_name = os.path.join(path, layer_json["digest"][7:])
         layer_artifact = Artifact.init_and_validate(layer_file_name)
         layer_artifact.save()
         blob = Blob(digest=layer_json["digest"])
@@ -60,7 +60,7 @@ def add_image_from_directory_to_repository(path, repository, tag):
         image and tag.
 
     """
-    manifest_path = "{}manifest.json".format(path)
+    manifest_path = os.path.join(path, "manifest.json")
     manifest_artifact = Artifact.init_and_validate(manifest_path)
     manifest_artifact.save()
     manifest_digest = "sha256:{}".format(manifest_artifact.sha256)
@@ -73,23 +73,27 @@ def add_image_from_directory_to_repository(path, repository, tag):
     ).save()
     tag = Tag(name=tag, tagged_manifest=manifest)
     tag.save()
+
     with repository.new_version() as new_repo_version:
-        new_repo_version.add_content(Manifest.objects.filter(pk=manifest.pk))
-        new_repo_version.add_content(Tag.objects.filter(pk=tag.pk))
-        with open(manifest_artifact.file.path, "r") as manifest_file:
-            manifest_json = json.load(manifest_file)
-            config_blob = get_or_create_blob(manifest_json["config"], manifest, path)
-            manifest.config_blob = config_blob
-            manifest.save()
-            new_repo_version.add_content(Blob.objects.filter(pk=config_blob.pk))
-            for layer in manifest_json["layers"]:
-                blob = get_or_create_blob(layer, manifest, path)
-                new_repo_version.add_content(Blob.objects.filter(pk=blob.pk))
+        manifest_json = json.load(manifest_artifact.file)
+        manifest_artifact.file.close()
+
+        config_blob = get_or_create_blob(manifest_json["config"], manifest, path)
+        manifest.config_blob = config_blob
+        manifest.save()
+
+        pks_to_add = []
+        for layer in manifest_json["layers"]:
+            pks_to_add.append(get_or_create_blob(layer, manifest, path).pk)
+
+        pks_to_add.extend([manifest.pk, tag.pk, config_blob.pk])
+        new_repo_version.add_content(Content.objects.filter(pk__in=pks_to_add))
+
     return new_repo_version
 
 
 def build_image_from_containerfile(
-    containerfile_pk=None, artifacts={}, repository_pk=None, tag=None
+    containerfile_pk=None, artifacts=None, repository_pk=None, tag=None
 ):
     """
     Builds an OCI container image from a Containerfile.
@@ -114,25 +118,43 @@ def build_image_from_containerfile(
     repository = ContainerRepository.objects.get(pk=repository_pk)
     name = str(uuid4())
     with tempfile.TemporaryDirectory(dir=".") as working_directory:
-        path = "{}/".format(working_directory.path)
+        working_directory = os.path.abspath(working_directory)
+        context_path = os.path.join(working_directory, "context")
+        os.makedirs(context_path)
         for key, val in artifacts.items():
             artifact = Artifact.objects.get(pk=key)
-            dirs = os.path.split(val)[0]
+            dest_path = os.path.join(context_path, val)
+            dirs = os.path.split(dest_path)[0]
             if dirs:
                 os.makedirs(dirs)
+            with open(dest_path, "wb") as dest:
+                shutil.copyfileobj(artifact.file, dest)
+
+            containerfile_path = os.path.join(working_directory, "Containerfile")
 
-            shutil.copy(artifact.file.path, "{}{}".format(path, val))
+        with open(containerfile_path, "wb") as dest:
+            shutil.copyfileobj(containerfile.file, dest)
         bud_cp = subprocess.run(
-            ["buildah", "bud", "-f", containerfile.file.path, "-t", name],
+            [
+                "podman",
+                "build",
+                "-f",
+                containerfile_path,
+                "-t",
+                name,
+                context_path,
+                "--isolation",
+                "rootless",
+            ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
         if bud_cp.returncode != 0:
             raise Exception(bud_cp.stderr)
-        image_dir = "{}image/".format(path)
+        image_dir = os.path.join(working_directory, "image")
         os.makedirs(image_dir)
         push_cp = subprocess.run(
-            ["buildah", "push", "-f", "oci", name, "dir:{}".format(image_dir)],
+            ["podman", "push", "-f", "oci", name, "dir:{}".format(image_dir)],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )

pulp_container/app/viewsets.py

@@ -861,8 +861,7 @@ def build_image(self, request, pk):
         tag = serializer.validated_data["tag"]
 
         artifacts = serializer.validated_data["artifacts"]
-        for artifact in artifacts:
-            artifact.touch()
+        Artifact.objects.filter(pk__in=artifacts.keys()).touch()
 
         result = dispatch(
             tasks.build_image_from_containerfile,

pulp_container/tests/functional/api/test_build_images.py

@@ -0,0 +1,63 @@
+import pytest
+
+from tempfile import NamedTemporaryFile
+
+from pulp_smash.pulp3.utils import (
+    gen_distribution,
+    gen_repo,
+)
+from pulp_smash.pulp3.bindings import monitor_task
+
+from pulpcore.client.pulp_container import (
+    ContainerContainerDistribution,
+    ContainerContainerRepository,
+)
+
+
+@pytest.fixture
+def containerfile_name():
+    """A fixture for a basic container file used for building images."""
+    with NamedTemporaryFile() as containerfile:
+        containerfile.write(
+            b"""FROM busybox:latest
+# Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter.
+COPY foo/bar/example.txt /tmp/inside-image.txt
+# Print the content of the file when the container starts
+CMD ["cat", "/tmp/inside-image.txt"]"""
+        )
+        containerfile.flush()
+        yield containerfile.name
+
+
+def test_build_image(
+    artifacts_api_client,
+    container_repository_api,
+    container_distribution_api,
+    gen_object_with_cleanup,
+    containerfile_name,
+    local_registry,
+):
+    """Test if a user can build an OCI image."""
+    with NamedTemporaryFile() as text_file:
+        text_file.write(b"some text")
+        text_file.flush()
+        artifact = gen_object_with_cleanup(artifacts_api_client, text_file.name)
+
+    repository = gen_object_with_cleanup(
+        container_repository_api, ContainerContainerRepository(**gen_repo())
+    )
+
+    artifacts = '{{"{}": "foo/bar/example.txt"}}'.format(artifact.pulp_href)
+    build_response = container_repository_api.build_image(
+        repository.pulp_href, containerfile=containerfile_name, artifacts=artifacts
+    )
+    monitor_task(build_response.task)
+
+    distribution = gen_object_with_cleanup(
+        container_distribution_api,
+        ContainerContainerDistribution(**gen_distribution(repository=repository.pulp_href)),
+    )
+
+    local_registry.pull(distribution.base_path)
+    image = local_registry.inspect(distribution.base_path)
+    assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"]

pulp_container/tests/functional/conftest.py

@@ -137,6 +137,11 @@ def tag_and_push(image_path, local_url, *args):
                 registry_client.rmi(local_image_path)
                 registry_client.logout(registry_name)
 
+        @staticmethod
+        def inspect(local_url):
+            local_image_path = "/".join([registry_name, local_url])
+            return registry_client.inspect(local_image_path)
+
     return _LocalRegistry()