update twoliter and buildsys to put RPMs into per-package sub-directories

tools/buildsys/src/args.rs

@@ -122,9 +122,6 @@ pub(crate) struct BuildPackageArgs {
     #[arg(long, env = "BUILDSYS_UPSTREAM_SOURCE_FALLBACK")]
     pub(crate) upstream_source_fallback: String,
 
-    #[arg(long, env = "CARGO_PKG_NAME")]
-    pub(crate) cargo_package_name: String,
-
     #[command(flatten)]
     pub(crate) common: Common,
 }

tools/buildsys/src/builder.rs

@@ -302,11 +302,8 @@ impl DockerBuild {
         manifest: &Manifest,
         image_features: HashSet<ImageFeature>,
     ) -> Result<Self> {
-        let package = if let Some(name_override) = manifest.info().package_name() {
-            name_override.clone()
-        } else {
-            args.cargo_package_name
-        };
+        let package = manifest.info().package_name();
+        let per_package_dir = format!("{}/{}", args.packages_dir.display(), package).into();
 
         Ok(Self {
             dockerfile: args.common.tools_dir.join("build.Dockerfile"),
@@ -321,9 +318,9 @@ impl DockerBuild {
                 &args.common.root_dir,
             ),
             root_dir: args.common.root_dir.clone(),
-            artifacts_dir: args.packages_dir,
+            artifacts_dir: per_package_dir,
             state_dir: args.common.state_dir,
-            artifact_name: package.clone(),
+            artifact_name: package.to_string(),
             common_build_args: CommonBuildArgs::new(
                 &args.common.root_dir,
                 args.common.sdk_image,
@@ -332,7 +329,7 @@ impl DockerBuild {
             ),
             target_build_args: TargetBuildArgs::Package(PackageBuildArgs {
                 image_features,
-                package,
+                package: package.to_string(),
                 package_dependencies: manifest.package_dependencies().context(error::GraphSnafu)?,
                 kit_dependencies: manifest.kit_dependencies().context(error::GraphSnafu)?,
                 publish_repo: args.publish_repo,

tools/buildsys/src/main.rs

@@ -203,11 +203,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
 
     // Package developer can override name of package if desired, e.g. to name package with
     // characters invalid in Cargo crate names
-    let package = if let Some(name_override) = manifest.info().package_name() {
-        name_override.clone()
-    } else {
-        args.cargo_package_name.clone()
-    };
+    let package = manifest.info().package_name();
     let spec = format!("{}.spec", package);
     println!("cargo:rerun-if-changed={}", spec);
 

tools/buildsys/src/manifest.rs

@@ -397,9 +397,13 @@ impl ManifestInfo {
         self.build_package().and_then(|b| b.external_files.as_ref())
     }
 
-    /// Convenience method to return the package name override, if any.
-    pub fn package_name(&self) -> Option<&String> {
-        self.build_package().and_then(|b| b.package_name.as_ref())
+    /// Convenience method to return the package name. If the manifest has an override in the
+    /// `package.metadata.build-package.package-name` key, it is returned, otherwise the Cargo
+    /// manifest name is returned from `package.name`.
+    pub fn package_name(&self) -> &str {
+        self.build_package()
+            .and_then(|b| b.package_name.as_deref())
+            .unwrap_or_else(|| self.manifest_name())
     }
 
     /// Convenience method to find whether the package is sensitive to variant changes.

twoliter/build.rs

@@ -36,6 +36,7 @@ fn main() {
     paths.copy_file("repack.Dockerfile");
     paths.copy_file("repack.Dockerfile.dockerignore");
     paths.copy_file("rpm2img");
+    paths.copy_file("rpm2kit");
     paths.copy_file("rpm2kmodkit");
     paths.copy_file("rpm2migrations");
     paths.copy_file("metadata.spec");

twoliter/embedded/Makefile.toml

@@ -750,6 +750,34 @@ cargo metadata \
 '''
 ]
 
+# This task is temporarily necessary due to changes in Twoliter that occurred in June 2024. A new
+# directory structure was introduced for build/rpms. Attempts to do an incremental build when the
+# build output was in the old directory structure would fail, so we needed a way to inform users
+# that they need to run a cargo make clean.
+#
+# Once the changed build/rpms structure has been around long enough (in, say, August 2024), this
+# task can be removed as we can assume that no lingering old incremental builds exist in the wild.
+#
+# TODO - remove this task sometime around, or after, August 2024.
+[tasks.check-rpm-structure]
+script_runner = "bash"
+script = [
+'''
+if [ ! -d "${BUILDSYS_PACKAGES_DIR}" ]; then
+  exit 0
+fi
+
+count=$(find "${BUILDSYS_PACKAGES_DIR}" -maxdepth 1 -name "*.rpm" | wc -l)
+
+if [ "${count}" != "0" ]; then
+    echo "ERROR: You have rpms in a flat structure in ${BUILDSYS_PACKAGES_DIR}"
+    echo "Twoliter and Buildsys have been updated to use a different RPM directory structure"
+    echo "PLEASE RUN: cargo make clean"
+    exit 1
+fi
+'''
+]
+
 # Builds a package including its build-time and runtime dependency packages.
 [tasks.build-package]
 dependencies = ["check-cargo-version", "fetch", "publish-setup", "fetch-licenses", "cargo-metadata"]

twoliter/embedded/build.Dockerfile

@@ -1,7 +1,8 @@
 # syntax=docker/dockerfile:1.4.3
-# This Dockerfile has two sections which are used to build rpm.spec packages and to create
-# Bottlerocket images, respectively. They are marked as Section 1 and Section 2. buildsys
-# uses Section 1 during build-package calls and Section 2 during build-variant calls.
+# This Dockerfile has three sections which are used to build rpm.spec packages, to create
+# kits, and to create Bottlerocket images, respectively. They are marked as Sections 1-3.
+# buildsys uses Section 1 during build-package calls, Section 2 during build-kit calls,
+# and Section 3 during build-variant calls.
 #
 # Several commands start with RUN --mount=target=/host, which mounts the docker build
 # context (which in practice is the root of the Bottlerocket repository) as a read-only
@@ -94,6 +95,8 @@ RUN \
 # Builds an RPM package from a spec file.
 FROM sdk AS rpmbuild
 ARG PACKAGE
+ARG PACKAGE_DEPENDENCIES
+ARG KIT_DEPENDENCIES
 ARG ARCH
 ARG NOCACHE
 ARG VARIANT
@@ -129,18 +132,26 @@ RUN \
 
 USER root
 RUN --mount=target=/host \
-    ln -s /host/build/rpms/*.rpm ./rpmbuild/RPMS \
-    && createrepo_c \
+    for pkg in ${PACKAGE_DEPENDENCIES} ; do \
+      ln -s /host/build/rpms/${pkg}/*.rpm ./rpmbuild/RPMS ; \
+    done && \
+    createrepo_c \
         -o ./rpmbuild/RPMS \
         -x '*-debuginfo-*.rpm' \
         -x '*-debugsource-*.rpm' \
         --no-database \
-        /host/build/rpms \
-    && cp .rpmmacros /etc/rpm/macros \
-    && dnf -y \
+        ./rpmbuild/RPMS && \
+    cp .rpmmacros /etc/rpm/macros && \
+    declare -a KIT_REPOS && \
+    for kit in ${KIT_DEPENDENCIES} ; do \
+      KIT_REPOS+=("--repofrompath=${kit},/host/build/kits/${kit}/${ARCH}" --enablerepo "${kit}") ; \
+    done && \
+    echo "${KIT_REPOS[@]}" && \
+    dnf -y \
         --disablerepo '*' \
         --repofrompath repo,./rpmbuild/RPMS \
         --enablerepo 'repo' \
+        "${KIT_REPOS[@]}" \
         --nogpgcheck \
         --forcearch "${ARCH}" \
         builddep rpmbuild/SPECS/${PACKAGE}.spec
@@ -167,13 +178,47 @@ FROM scratch AS package
 COPY --from=rpmbuild /home/builder/rpmbuild/RPMS/*/*.rpm /output/
 
 ############################################################################################
-# Section 2: The following build stages are used to create a Bottlerocket image once all of
-# the rpm files have been created by repeatedly using Section 1.
+# Section 2: The following build stages are used to create a Bottlerocket kit once all of
+# the rpm files have been created by repeatedly using Section 1. This process can occur more
+# than once because packages can depend on kits and those kits depend on packages that must
+# be built first.
 
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
-# Creates an RPM repository from packages created in Section 1.
+# Builds a kit from RPM packages.
+FROM sdk AS kitbuild
+ARG KIT
+ARG PACKAGE_DEPENDENCIES
+ARG ARCH
+ARG NOCACHE
+
+WORKDIR /home/builder
+USER builder
+
+RUN --mount=target=/host \
+    /host/build/tools/rpm2kit \
+        --packages-dir=/host/build/rpms \
+        --arch="${ARCH}" \
+        "${PACKAGE_DEPENDENCIES[@]/#/--package=}" \
+        --output-dir=/home/builder/output \
+    && echo ${NOCACHE}
+
+# Copies kit artifacts from the previous stage to their expected location so that buildsys
+# can find them and copy them out.
+FROM scratch AS kit
+COPY --from=kitbuild /home/builder/output/. /output/
+
+############################################################################################
+# Section 3: The following build stages are used to create a Bottlerocket image once all of
+# the rpm files have been created by repeatedly using Sections 1 and 2.
+
+# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
+# Creates an RPM repository from packages created in Section 1 and kits from Section 2.
 FROM sdk AS repobuild
+# The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section.
 ARG PACKAGES
+# The complete list of non-kit packages required by way of pure package-to-package dependencies.
+ARG PACKAGE_DEPENDENCIES
+ARG KIT_DEPENDENCIES
 ARG ARCH
 ARG NOCACHE
 
@@ -196,28 +241,36 @@ RUN --mount=target=/host \
 WORKDIR /root
 USER root
 RUN --mount=target=/host \
-    mkdir -p /local/rpms ./rpmbuild/RPMS \
-    && ln -s /host/build/rpms/*.rpm ./rpmbuild/RPMS \
-    && ln -s /home/builder/rpmbuild/RPMS/*/*.rpm ./rpmbuild/RPMS \
-    && createrepo_c \
+    mkdir -p ./rpmbuild/RPMS && \
+    for pkg in ${PACKAGE_DEPENDENCIES} ; do \
+      ln -s /host/build/rpms/${pkg}/*.rpm ./rpmbuild/RPMS ; \
+    done && \
+    ln -s /home/builder/rpmbuild/RPMS/*/*.rpm ./rpmbuild/RPMS && \
+    createrepo_c \
         -o ./rpmbuild/RPMS \
         -x '*-debuginfo-*.rpm' \
         -x '*-debugsource-*.rpm' \
         --no-database \
-        ./rpmbuild/RPMS \
-    && echo '%_dbpath %{_sharedstatedir}/rpm' >> /etc/rpm/macros \
-    && dnf -y \
+        ./rpmbuild/RPMS && \
+    echo '%_dbpath %{_sharedstatedir}/rpm' >> /etc/rpm/macros && \
+    declare -a KIT_REPOS && \
+    for kit in ${KIT_DEPENDENCIES} ; do \
+      KIT_REPOS+=("--repofrompath=${kit},/host/build/kits/${kit}/${ARCH}" --enablerepo "${kit}") ; \
+    done && \
+    dnf -y \
         --disablerepo '*' \
         --repofrompath repo,./rpmbuild/RPMS \
         --enablerepo 'repo' \
+        "${KIT_REPOS[@]}" \
         --nogpgcheck \
         --downloadonly \
         --downloaddir . \
         --forcearch "${ARCH}" \
-        install $(printf "bottlerocket-%s\n" metadata ${PACKAGES}) \
-    && mv *.rpm /local/rpms \
-    && createrepo_c /local/rpms \
-    && echo ${NOCACHE}
+        install $(printf "bottlerocket-%s\n" metadata ${PACKAGES}) && \
+    mkdir -p /local/rpms && \
+    mv *.rpm /local/rpms && \
+    createrepo_c /local/rpms && \
+    echo ${NOCACHE}
 
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
 # Builds a Bottlerocket image.
@@ -288,7 +341,7 @@ WORKDIR /root
 USER root
 RUN --mount=target=/host \
     mkdir -p /local/migrations \
-    && find /host/build/rpms/ -maxdepth 1 -type f \
+    && find /host/build/rpms/os/ -maxdepth 1 -type f \
         -name "bottlerocket-migrations-*.rpm" \
         -not -iname '*debuginfo*' \
         -exec cp '{}' '/local/migrations/' ';' \
@@ -300,6 +353,7 @@ RUN --mount=target=/host \
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
 # Creates an archive of kernel development sources and toolchain.
 FROM repobuild as kmodkitbuild
+# The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section.
 ARG PACKAGES
 ARG ARCH
 ARG VERSION_ID
@@ -314,7 +368,7 @@ WORKDIR /tmp
 RUN --mount=target=/host \
     mkdir -p /local/archives \
     && KERNEL="$(printf "%s\n" ${PACKAGES} | awk '/^kernel-/{print $1}')" \
-    && find /host/build/rpms/ -maxdepth 1 -type f \
+    && find /host/build/rpms/${KERNEL}/ -maxdepth 1 -type f \
         -name "bottlerocket-${KERNEL}-archive-*.rpm" \
         -exec cp '{}' '/local/archives/' ';' \
     && /host/build/tools/rpm2kmodkit \

twoliter/embedded/build.Dockerfile.dockerignore

@@ -6,6 +6,10 @@
 !/build/rpms/*.rpm
 /build/rpms/*-debuginfo-*.rpm
 /build/rpms/*-debugsource-*.rpm
+!/build/rpms/*/*.rpm
+/build/rpms/*/*-debuginfo-*.rpm
+/build/rpms/*/*-debugsource-*.rpm
+!/build/kits/
 !/build/tools/*
 **/target/*
 /sbkeys

twoliter/embedded/rpm2kit

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Create a kit from RPM package inputs.
+set -eu -o pipefail
+
+declare -a PACKAGES
+
+for opt in "$@"; do
+   optarg="$(expr "${opt}" : '[^=]*=\(.*\)')"
+   case "${opt}" in
+      --packages-dir=*) PACKAGES_DIR="${optarg}" ;;
+      --package=*) PACKAGES+=("${optarg}") ;;
+      --output-dir=*) OUTPUT_DIR="${optarg}" ;;
+   esac
+done
+
+KIT_DIR="${OUTPUT_DIR}/${ARCH}"
+rm -rf "${KIT_DIR}"
+
+mkdir -p "${KIT_DIR}/Packages"
+for pkg in ${PACKAGES} ; do
+  find "${PACKAGES_DIR}/${pkg}" \
+    -mindepth 1 \
+    -maxdepth 1 \
+    ! -name '*-debuginfo-*' \
+    ! -name '*-debugsource-*' \
+    -size +0c \
+    -exec install -p -m 0644 -t "${KIT_DIR}/Packages" {} \+
+done
+
+createrepo_c "${KIT_DIR}"
+dnf --disablerepo '*' --repofrompath "kit,file:///${KIT_DIR}" repoquery --all

twoliter/src/tools.rs

@@ -115,6 +115,7 @@ async fn test_install_tools() {
     assert!(toolsdir.join("repack.Dockerfile").is_file());
     assert!(toolsdir.join("repack.Dockerfile.dockerignore").is_file());
     assert!(toolsdir.join("rpm2img").is_file());
+    assert!(toolsdir.join("rpm2kit").is_file());
     assert!(toolsdir.join("rpm2kmodkit").is_file());
     assert!(toolsdir.join("rpm2migrations").is_file());