From a6c2103cbf95739a23db6950e7ca32e0725401c7 Mon Sep 17 00:00:00 2001 From: Lubos Mjachky Date: Mon, 16 May 2022 15:59:49 +0200 Subject: [PATCH] Add a test for building OCI images closes #461 --- CHANGES/461.bugfix | 1 + docs/installation.rst | 19 ++---- docs/workflows/build-containerfile.rst | 2 - pulp_container/app/serializers.py | 24 +++---- pulp_container/app/tasks/builder.py | 64 +++++++++++++------ pulp_container/app/viewsets.py | 3 +- .../tests/functional/api/test_build_images.py | 63 ++++++++++++++++++ pulp_container/tests/functional/conftest.py | 5 ++ 8 files changed, 129 insertions(+), 52 deletions(-) create mode 100644 CHANGES/461.bugfix create mode 100644 pulp_container/tests/functional/api/test_build_images.py diff --git a/CHANGES/461.bugfix b/CHANGES/461.bugfix new file mode 100644 index 000000000..6a88584b5 --- /dev/null +++ b/CHANGES/461.bugfix @@ -0,0 +1 @@ +Fixed the machinery for building OCI images. diff --git a/docs/installation.rst b/docs/installation.rst index 80a735c44..88c50179b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -69,16 +69,9 @@ Run Services sudo systemctl restart pulpcore-worker@1 sudo systemctl restart pulpcore-worker@2 -Enable OCI Container Image building ------------------------------------ - -Pulp container plugin can be used to build an OCI format image from a Containerfile. The plugin uses -`buildah `_ to build the container image. Buildah 1.14+ -must be installed on the same machine that is running pulpcore-worker processes. - -The pulpcore-worker processes needs to have `/usr/bin/` in its `PATH`. The user that is running -pulpcore-worker process needs to be able to manage subordinate user ids and group ids. The range of -subordinate user ids is specified in `/etc/subuid` and the range of subordinate group ids is -specified in `/etc/subgid`. More details can be found in `buildah documentation `_. +OCI Container Image building +---------------------------- + +The plugin can be used to build an OCI format image from a Containerfile. The plugin uses podman +to build containers. Refer to `podman-build documentation `_ +for more details. diff --git a/docs/workflows/build-containerfile.rst b/docs/workflows/build-containerfile.rst index e0d7cdf97..fb48df535 100644 --- a/docs/workflows/build-containerfile.rst +++ b/docs/workflows/build-containerfile.rst @@ -8,8 +8,6 @@ Build an OCI image from a Containerfile All container build APIs are tech preview in Pulp Container 2.1. Backwards compatibility when upgrading is not guaranteed. - `buildah` needs to be installed to enable image building. - Users can add new images to a container repository by uploading a Containerfile. The syntax for Containerfile is the same as for a Dockerfile. The same REST API endpoint also accepts a JSON string that maps artifacts in Pulp to a filename. Any artifacts passed in are available inside the diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 8d0b3123c..87393a713 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -496,27 +496,21 @@ class CopySerializer(ValidateFieldsMixin, serializers.Serializer): ) def validate(self, data): - """Ensure that source_repository or source_rpository_version is pass, but not both.""" + """Ensure that source_repository or source_repository_version is passed, but not both.""" data = super().validate(data) repository = data.pop("source_repository", None) repository_version = data.get("source_repository_version") if not repository and not repository_version: raise serializers.ValidationError( - _("Either the 'repository' or 'repository_version' need to be specified") + _("Either the 'repository' or 'repository_version' needs to be specified") ) elif not repository and repository_version: return data elif repository and not repository_version: - version = repository.latest_version() - if version: - new_data = {"source_repository_version": version} - new_data.update(data) - return new_data - else: - raise serializers.ValidationError( - detail=_("Source repository has no version available to copy content from") - ) + new_data = {"source_repository_version": repository.latest_version()} + new_data.update(data) + return new_data raise serializers.ValidationError( _( "Either the 'repository' or 'repository_version' need to be specified " @@ -640,10 +634,12 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer): lookup_field="pk", view_name="artifacts-detail", queryset=Artifact.objects.all(), - help_text=_("Artifact representing the Containerfile that should be used to run buildah."), + help_text=_( + "Artifact representing the Containerfile that should be used to run podman-build." + ), ) containerfile = serializers.FileField( - help_text=_("An uploaded Containerfile that should be used to run buildah."), + help_text=_("An uploaded Containerfile that should be used to run podman-build."), required=False, ) tag = serializers.CharField( @@ -693,7 +689,7 @@ def validate(self, data): try: artifact = artifactfield.run_validation(data=url) artifact.touch() - artifacts[artifact.pk] = relative_path + artifacts[str(artifact.pk)] = relative_path except serializers.ValidationError as e: # Append the URL of missing Artifact to the error message e.detail[0] = "%s %s" % (e.detail[0], url) diff --git a/pulp_container/app/tasks/builder.py b/pulp_container/app/tasks/builder.py index 9509d8bb3..bc24aeb5d 100644 --- a/pulp_container/app/tasks/builder.py +++ b/pulp_container/app/tasks/builder.py @@ -13,7 +13,7 @@ Tag, ) from pulp_container.constants import MEDIA_TYPE -from pulpcore.plugin.models import Artifact, ContentArtifact +from pulpcore.plugin.models import Artifact, ContentArtifact, Content def get_or_create_blob(layer_json, manifest, path): @@ -33,7 +33,7 @@ def get_or_create_blob(layer_json, manifest, path): blob = Blob.objects.get(digest=layer_json["digest"]) blob.touch() except Blob.DoesNotExist: - layer_file_name = "{}{}".format(path, layer_json["digest"][7:]) + layer_file_name = os.path.join(path, layer_json["digest"][7:]) layer_artifact = Artifact.init_and_validate(layer_file_name) layer_artifact.save() blob = Blob(digest=layer_json["digest"]) @@ -60,7 +60,7 @@ def add_image_from_directory_to_repository(path, repository, tag): image and tag. """ - manifest_path = "{}manifest.json".format(path) + manifest_path = os.path.join(path, "manifest.json") manifest_artifact = Artifact.init_and_validate(manifest_path) manifest_artifact.save() manifest_digest = "sha256:{}".format(manifest_artifact.sha256) @@ -73,23 +73,27 @@ def add_image_from_directory_to_repository(path, repository, tag): ).save() tag = Tag(name=tag, tagged_manifest=manifest) tag.save() + with repository.new_version() as new_repo_version: - new_repo_version.add_content(Manifest.objects.filter(pk=manifest.pk)) - new_repo_version.add_content(Tag.objects.filter(pk=tag.pk)) - with open(manifest_artifact.file.path, "r") as manifest_file: - manifest_json = json.load(manifest_file) - config_blob = get_or_create_blob(manifest_json["config"], manifest, path) - manifest.config_blob = config_blob - manifest.save() - new_repo_version.add_content(Blob.objects.filter(pk=config_blob.pk)) - for layer in manifest_json["layers"]: - blob = get_or_create_blob(layer, manifest, path) - new_repo_version.add_content(Blob.objects.filter(pk=blob.pk)) + manifest_json = json.load(manifest_artifact.file) + manifest_artifact.file.close() + + config_blob = get_or_create_blob(manifest_json["config"], manifest, path) + manifest.config_blob = config_blob + manifest.save() + + pks_to_add = [] + for layer in manifest_json["layers"]: + pks_to_add.append(get_or_create_blob(layer, manifest, path).pk) + + pks_to_add.extend([manifest.pk, tag.pk, config_blob.pk]) + new_repo_version.add_content(Content.objects.filter(pk__in=pks_to_add)) + return new_repo_version def build_image_from_containerfile( - containerfile_pk=None, artifacts={}, repository_pk=None, tag=None + containerfile_pk=None, artifacts=None, repository_pk=None, tag=None ): """ Builds an OCI container image from a Containerfile. @@ -114,25 +118,43 @@ def build_image_from_containerfile( repository = ContainerRepository.objects.get(pk=repository_pk) name = str(uuid4()) with tempfile.TemporaryDirectory(dir=".") as working_directory: - path = "{}/".format(working_directory.path) + working_directory = os.path.abspath(working_directory) + context_path = os.path.join(working_directory, "context") + os.makedirs(context_path) for key, val in artifacts.items(): artifact = Artifact.objects.get(pk=key) - dirs = os.path.split(val)[0] + dest_path = os.path.join(context_path, val) + dirs = os.path.split(dest_path)[0] if dirs: os.makedirs(dirs) + with open(dest_path, "wb") as dest: + shutil.copyfileobj(artifact.file, dest) + + containerfile_path = os.path.join(working_directory, "Containerfile") - shutil.copy(artifact.file.path, "{}{}".format(path, val)) + with open(containerfile_path, "wb") as dest: + shutil.copyfileobj(containerfile.file, dest) bud_cp = subprocess.run( - ["buildah", "bud", "-f", containerfile.file.path, "-t", name], + [ + "podman", + "build", + "-f", + containerfile_path, + "-t", + name, + context_path, + "--isolation", + "rootless", + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) if bud_cp.returncode != 0: raise Exception(bud_cp.stderr) - image_dir = "{}image/".format(path) + image_dir = os.path.join(working_directory, "image") os.makedirs(image_dir) push_cp = subprocess.run( - ["buildah", "push", "-f", "oci", name, "dir:{}".format(image_dir)], + ["podman", "push", "-f", "oci", name, "dir:{}".format(image_dir)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) diff --git a/pulp_container/app/viewsets.py b/pulp_container/app/viewsets.py index 695722b03..62776406e 100644 --- a/pulp_container/app/viewsets.py +++ b/pulp_container/app/viewsets.py @@ -861,8 +861,7 @@ def build_image(self, request, pk): tag = serializer.validated_data["tag"] artifacts = serializer.validated_data["artifacts"] - for artifact in artifacts: - artifact.touch() + Artifact.objects.filter(pk__in=artifacts.keys()).touch() result = dispatch( tasks.build_image_from_containerfile, diff --git a/pulp_container/tests/functional/api/test_build_images.py b/pulp_container/tests/functional/api/test_build_images.py new file mode 100644 index 000000000..52e154907 --- /dev/null +++ b/pulp_container/tests/functional/api/test_build_images.py @@ -0,0 +1,63 @@ +import pytest + +from tempfile import NamedTemporaryFile + +from pulp_smash.pulp3.utils import ( + gen_distribution, + gen_repo, +) +from pulp_smash.pulp3.bindings import monitor_task + +from pulpcore.client.pulp_container import ( + ContainerContainerDistribution, + ContainerContainerRepository, +) + + +@pytest.fixture +def containerfile_name(): + """A fixture for a basic container file used for building images.""" + with NamedTemporaryFile() as containerfile: + containerfile.write( + b"""FROM busybox:latest +# Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter. +COPY foo/bar/example.txt /tmp/inside-image.txt +# Print the content of the file when the container starts +CMD ["cat", "/tmp/inside-image.txt"]""" + ) + containerfile.flush() + yield containerfile.name + + +def test_build_image( + artifacts_api_client, + container_repository_api, + container_distribution_api, + gen_object_with_cleanup, + containerfile_name, + local_registry, +): + """Test if a user can build an OCI image.""" + with NamedTemporaryFile() as text_file: + text_file.write(b"some text") + text_file.flush() + artifact = gen_object_with_cleanup(artifacts_api_client, text_file.name) + + repository = gen_object_with_cleanup( + container_repository_api, ContainerContainerRepository(**gen_repo()) + ) + + artifacts = '{{"{}": "foo/bar/example.txt"}}'.format(artifact.pulp_href) + build_response = container_repository_api.build_image( + repository.pulp_href, containerfile=containerfile_name, artifacts=artifacts + ) + monitor_task(build_response.task) + + distribution = gen_object_with_cleanup( + container_distribution_api, + ContainerContainerDistribution(**gen_distribution(repository=repository.pulp_href)), + ) + + local_registry.pull(distribution.base_path) + image = local_registry.inspect(distribution.base_path) + assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"] diff --git a/pulp_container/tests/functional/conftest.py b/pulp_container/tests/functional/conftest.py index f56a5ce8e..864aad0c8 100644 --- a/pulp_container/tests/functional/conftest.py +++ b/pulp_container/tests/functional/conftest.py @@ -137,6 +137,11 @@ def tag_and_push(image_path, local_url, *args): registry_client.rmi(local_image_path) registry_client.logout(registry_name) + @staticmethod + def inspect(local_url): + local_image_path = "/".join([registry_name, local_url]) + return registry_client.inspect(local_image_path) + return _LocalRegistry()