cachi2: postprocess

Postprocssing plugin to take cachi2 generated dependencies and generate
expected metadata for OSBS and prepare sources into build dirs.

Signed-off-by: Martin Basti <mbasti@redhat.com>

atomic_reactor/cli/parser.py

@@ -101,6 +101,13 @@ def parse_args(args: Optional[Sequence[str]] = None) -> dict:
     )
     binary_container_cachi2_init.set_defaults(func=task.binary_container_cachi2_init)
 
+    binary_container_cachi2_postprocess = tasks.add_parser(
+        "binary-container-cachi2-postprocess",
+        help="binary container cachi2 init step",
+        description="Execute binary container cachi2 postprocess step.",
+    )
+    binary_container_cachi2_postprocess.set_defaults(func=task.binary_container_cachi2_postprocess)
+
     binary_container_prebuild = tasks.add_parser(
         "binary-container-prebuild",
         help="binary container pre-build step",

atomic_reactor/cli/task.py

@@ -7,7 +7,7 @@
 """
 from atomic_reactor.tasks.binary import (BinaryExitTask, BinaryPostBuildTask, BinaryPreBuildTask,
                                          BinaryInitTask, BinaryCachitoTask,
-                                         BinaryCachi2InitTask,
+                                         BinaryCachi2InitTask, BinaryCachi2PostprocessTask,
                                          InitTaskParams, BinaryExitTaskParams)
 from atomic_reactor.tasks.binary_container_build import BinaryBuildTask, BinaryBuildTaskParams
 from atomic_reactor.tasks.clone import CloneTask
@@ -76,6 +76,16 @@ def binary_container_cachi2_init(task_args: dict):
     return task.run(init_build_dirs=True)
 
 
+def binary_container_cachi2_postprocess(task_args: dict):
+    """Run binary container Cachi2 postprocess step.
+
+    :param task_args: CLI arguments for a binary-container-cachi2-postprocess task
+    """
+    params = TaskParams.from_cli_args(task_args)
+    task = BinaryCachi2PostprocessTask(params)
+    return task.run(init_build_dirs=True)
+
+
 def binary_container_prebuild(task_args: dict):
     """Run binary container pre-build steps.
 

atomic_reactor/constants.py

@@ -113,6 +113,7 @@
 PLUGIN_GENERATE_SBOM = 'generate_sbom'
 PLUGIN_RPMQA = 'all_rpm_packages'
 PLUGIN_CACHI2_INIT = "cachi2_init"
+PLUGIN_CACHI2_POSTPROCESS = "cachi2_postprocess"
 
 # some shared dict keys for build metadata that gets recorded with koji.
 # for consistency of metadata in historical builds, these values basically cannot change.

atomic_reactor/plugins/cachi2_postprocess.py

@@ -0,0 +1,242 @@
+"""
+Copyright (c) 2024 Red Hat, Inc
+All rights reserved.
+
+This software may be modified and distributed under the terms
+of the BSD license. See the LICENSE file for details.
+"""
+import functools
+import json
+import os.path
+import shlex
+from dataclasses import dataclass
+from pathlib import Path
+from shutil import copytree
+from typing import Any, Optional, List, Dict
+
+from atomic_reactor.constants import (
+    CACHITO_ENV_ARG_ALIAS,
+    CACHITO_ENV_FILENAME,
+    PLUGIN_CACHI2_INIT,
+    PLUGIN_CACHI2_POSTPROCESS,
+    REMOTE_SOURCE_DIR,
+    REMOTE_SOURCE_JSON_FILENAME,
+    REMOTE_SOURCE_TARBALL_FILENAME,
+    REMOTE_SOURCE_JSON_ENV_FILENAME,
+)
+from atomic_reactor.dirs import BuildDir
+from atomic_reactor.plugin import Plugin
+
+from atomic_reactor.utils.cachi2 import generate_request_json
+
+
+@dataclass(frozen=True)
+class Cachi2RemoteSource:
+    """Represents a processed remote source.
+
+    name: the name that identifies this remote source (if multiple remote sources were used)
+    json_data: subset of the JSON representation of the Cachito request (source_request_to_json)
+    build_args: environment variables for this remote source
+    tarball_path: the path of the tarball downloaded from Cachito
+    """
+
+    name: Optional[str]
+    json_data: dict
+    json_env_data: List[Dict[str, str]]
+    tarball_path: Path
+    sources_path: Path
+
+    @classmethod
+    def tarball_filename(cls, name: Optional[str]):
+        if name:
+            return f"remote-source-{name}.tar.gz"
+        else:
+            return REMOTE_SOURCE_TARBALL_FILENAME
+
+    @classmethod
+    def json_filename(cls, name: Optional[str]):
+        if name:
+            return f"remote-source-{name}.json"
+        else:
+            return REMOTE_SOURCE_JSON_FILENAME
+
+    @classmethod
+    def json_env_filename(cls, name: Optional[str]):
+        if name:
+            return f"remote-source-{name}.env.json"
+        else:
+            return REMOTE_SOURCE_JSON_ENV_FILENAME
+
+    @property
+    def build_args(self) -> Dict[str, str]:
+
+        return {
+            env_var['name']: env_var['value']
+            for env_var in self.json_env_data
+        }
+
+
+class Cachi2PostprocessPlugin(Plugin):
+    """Postprocess cachi2 results
+
+    This plugin will postprocess cachi2 results and provide required metadata
+    """
+
+    key = PLUGIN_CACHI2_POSTPROCESS
+    is_allowed_to_fail = False
+    REMOTE_SOURCE = "unpacked_remote_sources"
+
+    def __init__(self, workflow):
+        """
+        :param workflow: DockerBuildWorkflow instance
+
+        """
+        super(Cachi2PostprocessPlugin, self).__init__(workflow)
+        self._osbs = None
+        self.single_remote_source_params = self.workflow.source.config.remote_source
+        self.multiple_remote_sources_params = self.workflow.source.config.remote_sources
+        self.init_plugin_data = self.workflow.data.plugins_results.get(PLUGIN_CACHI2_INIT)
+
+    def run(self) -> Optional[List[Dict[str, Any]]]:
+        if not self.init_plugin_data:
+            self.log.info('Aborting plugin execution: no cachi2 data provided')
+            return None
+
+        if not (self.single_remote_source_params or self.multiple_remote_sources_params):
+            self.log.info('Aborting plugin execution: missing remote source configuration')
+            return None
+
+        processed_remote_sources = self.postprocess_remote_sources()
+        self.inject_remote_sources(processed_remote_sources)
+
+        return [
+            self.remote_source_to_output(remote_source)
+            for remote_source in processed_remote_sources
+        ]
+
+    def postprocess_remote_sources(self) -> List[Cachi2RemoteSource]:
+        """Process remote source requests and return information about the processed sources."""
+
+        processed_remote_sources = []
+
+        for remote_source in self.init_plugin_data:
+
+            json_env_path = os.path.join(remote_source['source_path'], 'cachi2.env.json')
+            with open(json_env_path, 'r') as json_f:
+                json_env_data = json.load(json_f)
+
+            sbom_path = os.path.join(remote_source['source_path'], 'deps', 'bom.json')
+            with open(sbom_path, 'r') as sbom_f:
+                sbom_data = json.load(sbom_f)
+
+            remote_source_obj = Cachi2RemoteSource(
+                name=remote_source['name'] or '',
+                tarball_path=os.path.join(remote_source['source_path'], 'remote-source.tar.gz'),
+                sources_path=Path(remote_source['source_path']),
+                json_data=generate_request_json(remote_source, sbom_data, json_env_data),
+                json_env_data=json_env_data,
+            )
+            processed_remote_sources.append(remote_source_obj)
+        return processed_remote_sources
+
+    def inject_remote_sources(self, remote_sources: List[Cachi2RemoteSource]) -> None:
+        """Inject processed remote sources into build dirs and add build args to workflow."""
+        inject_sources = functools.partial(self.inject_into_build_dir, remote_sources)
+        self.workflow.build_dir.for_all_platforms_copy(inject_sources)
+
+        # For single remote_source workflow, inject all build args directly
+        if self.single_remote_source_params:
+            self.workflow.data.buildargs.update(remote_sources[0].build_args)
+
+        self.add_general_buildargs()
+
+    def inject_into_build_dir(
+        self, remote_sources: List[Cachi2RemoteSource], build_dir: BuildDir,
+    ) -> List[Path]:
+        """Inject processed remote sources into a build directory.
+
+        For each remote source, create a dedicated directory, unpack the downloaded tarball
+        into it and inject the configuration files and an environment file.
+
+        Return a list of the newly created directories.
+        """
+        created_dirs = []
+
+        for remote_source in remote_sources:
+            dest_dir = build_dir.path.joinpath(self.REMOTE_SOURCE, remote_source.name or "")
+
+            if dest_dir.exists():
+                raise RuntimeError(
+                    f"Conflicting path {dest_dir.relative_to(build_dir.path)} already exists "
+                    "in the dist-git repository"
+                )
+
+            dest_dir.mkdir(parents=True)
+            created_dirs.append(dest_dir)
+
+            # copy app and deps generated by cachito into build_dir
+            # TODO: reflink?
+            copytree(remote_source['source_path']/'app', dest_dir/'app', symlinks=True)
+            copytree(remote_source['source_path']/'deps', dest_dir/'deps', symlinks=True)
+
+            # Create cachito.env file with environment variables received from cachito request
+            self.generate_cachito_env_file(dest_dir, remote_source.build_args)
+
+        return created_dirs
+
+    def remote_source_to_output(self, remote_source: Cachi2RemoteSource) -> Dict[str, Any]:
+        """Convert a processed remote source to a dict to be used as output of this plugin."""
+
+        return {
+            "name": remote_source.name,
+            "remote_source_json": {
+                "json": remote_source.json_data,
+                "filename": Cachi2RemoteSource.json_filename(remote_source.name),
+            },
+            "remote_source_json_env": {
+                "json": remote_source.json_env_data,
+                "filename": Cachi2RemoteSource.json_env_filename(remote_source.name),
+            },
+            "remote_source_tarball": {
+                "filename": remote_source.tarball_path.name,
+                "path": str(remote_source.tarball_path),
+            },
+        }
+
+    def generate_cachito_env_file(self, dest_dir: Path, build_args: Dict[str, str]) -> None:
+        """
+        Generate cachito.env file with exported environment variables received from
+        cachito request.
+
+        :param dest_dir: destination directory for env file
+        :param build_args: build arguments to set
+        """
+        self.log.info('Creating %s file with environment variables '
+                      'received from cachi2', CACHITO_ENV_FILENAME)
+
+        # Use dedicated dir in container build workdir for cachito.env
+        abs_path = dest_dir / CACHITO_ENV_FILENAME
+        with open(abs_path, 'w') as f:
+            f.write('#!/bin/bash\n')
+            for env_var, value in build_args.items():
+                f.write('export {}={}\n'.format(env_var, shlex.quote(value)))
+
+    def add_general_buildargs(self) -> None:
+        """Adds general build arguments
+
+        To copy the sources into the build image, Dockerfile should contain
+        COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR
+        or COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR
+        """
+        if self.multiple_remote_sources_params:
+            args_for_dockerfile_to_add = {
+                'REMOTE_SOURCES': self.REMOTE_SOURCE,
+                'REMOTE_SOURCES_DIR': REMOTE_SOURCE_DIR,
+            }
+        else:
+            args_for_dockerfile_to_add = {
+                'REMOTE_SOURCE': self.REMOTE_SOURCE,
+                'REMOTE_SOURCE_DIR': REMOTE_SOURCE_DIR,
+                CACHITO_ENV_ARG_ALIAS: os.path.join(REMOTE_SOURCE_DIR, CACHITO_ENV_FILENAME),
+            }
+        self.workflow.data.buildargs.update(args_for_dockerfile_to_add)

atomic_reactor/tasks/binary.py

@@ -14,6 +14,7 @@
 from atomic_reactor import inner
 from atomic_reactor.constants import (
     PLUGIN_CACHI2_INIT,
+    PLUGIN_CACHI2_POSTPROCESS,
     DOCKERFILE_FILENAME,
 )
 from atomic_reactor.tasks import plugin_based
@@ -89,6 +90,15 @@ class BinaryCachi2InitTask(plugin_based.PluginBasedTask[TaskParams]):
     ]
 
 
+class BinaryCachi2PostprocessTask(plugin_based.PluginBasedTask[TaskParams]):
+    """Binary container Cachi2 postprocess task."""
+
+    task_name = 'binary_container_cachi2_postprocess'
+    plugins_conf = [
+        {"name": PLUGIN_CACHI2_POSTPROCESS},
+    ]
+
+
 class BinaryPreBuildTask(plugin_based.PluginBasedTask[TaskParams]):
     """Binary container pre-build task."""
 

atomic_reactor/utils/cachi2.py

@@ -144,7 +144,7 @@ def generate_request_json(
         "pkg_managers": remote_source.get("pkg_managers", []),
         "ref": remote_source["ref"],
         "repo": remote_source["repo"],
-        "environment_variables": {key: val["value"] for key, val in remote_source_env_json.items()},
+        "environment_variables": {env['name']: env["value"] for env in remote_source_env_json},
         "flags": remote_source.get("flags", []),
         "packages": [],  # this will be always empty cachi2 doesn't provide nested deps
     }

tekton/tasks/binary-container-cachi2.yaml

@@ -117,3 +117,23 @@ spec:
           # single SBOM is the final SBOM
           cp "${SBOMS[0]}" "${CACHI2_DIR}/bom.json"
         fi
+ - name: binary-container-cachi2-postprocess
+      image: $(params.osbs-image)
+      workingDir: $(workspaces.ws-home-dir.path)
+      resources:
+        requests:
+          memory: 512Mi
+          cpu: 250m
+        limits:
+          memory: 1Gi
+          cpu: 395m
+      script: |
+        set -x
+        atomic-reactor -v task \
+          --user-params='$(params.user-params)' \
+          --build-dir=$(workspaces.ws-build-dir.path) \
+          --context-dir=$(workspaces.ws-context-dir.path) \
+          --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml \
+          --namespace=$(context.taskRun.namespace) \
+          --pipeline-run-name="$(params.pipeline-run-name)" \
+          binary-container-cachi2-postprocess