diff --git a/tools/buildsys/src/args.rs b/tools/buildsys/src/args.rs index dab1f5a01..6965c9ab0 100644 --- a/tools/buildsys/src/args.rs +++ b/tools/buildsys/src/args.rs @@ -122,9 +122,6 @@ pub(crate) struct BuildPackageArgs { #[arg(long, env = "BUILDSYS_UPSTREAM_SOURCE_FALLBACK")] pub(crate) upstream_source_fallback: String, - #[arg(long, env = "CARGO_PKG_NAME")] - pub(crate) cargo_package_name: String, - #[command(flatten)] pub(crate) common: Common, } diff --git a/tools/buildsys/src/builder.rs b/tools/buildsys/src/builder.rs index d1b5fc8ed..9b998a849 100644 --- a/tools/buildsys/src/builder.rs +++ b/tools/buildsys/src/builder.rs @@ -302,11 +302,8 @@ impl DockerBuild { manifest: &Manifest, image_features: HashSet, ) -> Result { - let package = if let Some(name_override) = manifest.info().package_name() { - name_override.clone() - } else { - args.cargo_package_name - }; + let package = manifest.info().package_name(); + let per_package_dir = format!("{}/{}", args.packages_dir.display(), package).into(); Ok(Self { dockerfile: args.common.tools_dir.join("build.Dockerfile"), @@ -321,9 +318,9 @@ impl DockerBuild { &args.common.root_dir, ), root_dir: args.common.root_dir.clone(), - artifacts_dir: args.packages_dir, + artifacts_dir: per_package_dir, state_dir: args.common.state_dir, - artifact_name: package.clone(), + artifact_name: package.to_string(), common_build_args: CommonBuildArgs::new( &args.common.root_dir, args.common.sdk_image, @@ -332,7 +329,7 @@ impl DockerBuild { ), target_build_args: TargetBuildArgs::Package(PackageBuildArgs { image_features, - package, + package: package.to_string(), package_dependencies: manifest.package_dependencies().context(error::GraphSnafu)?, kit_dependencies: manifest.kit_dependencies().context(error::GraphSnafu)?, publish_repo: args.publish_repo, diff --git a/tools/buildsys/src/main.rs b/tools/buildsys/src/main.rs index 0d20eea34..58f69935b 100644 --- a/tools/buildsys/src/main.rs +++ b/tools/buildsys/src/main.rs @@ -203,11 +203,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> { // Package developer can override name of package if desired, e.g. to name package with // characters invalid in Cargo crate names - let package = if let Some(name_override) = manifest.info().package_name() { - name_override.clone() - } else { - args.cargo_package_name.clone() - }; + let package = manifest.info().package_name(); let spec = format!("{}.spec", package); println!("cargo:rerun-if-changed={}", spec); diff --git a/tools/buildsys/src/manifest.rs b/tools/buildsys/src/manifest.rs index 8fbb0299c..040349dd7 100644 --- a/tools/buildsys/src/manifest.rs +++ b/tools/buildsys/src/manifest.rs @@ -397,9 +397,13 @@ impl ManifestInfo { self.build_package().and_then(|b| b.external_files.as_ref()) } - /// Convenience method to return the package name override, if any. - pub fn package_name(&self) -> Option<&String> { - self.build_package().and_then(|b| b.package_name.as_ref()) + /// Convenience method to return the package name. If the manifest has an override in the + /// `package.metadata.build-package.package-name` key, it is returned, otherwise the Cargo + /// manifest name is returned from `package.name`. + pub fn package_name(&self) -> &str { + self.build_package() + .and_then(|b| b.package_name.as_deref()) + .unwrap_or_else(|| self.manifest_name()) } /// Convenience method to find whether the package is sensitive to variant changes. diff --git a/twoliter/build.rs b/twoliter/build.rs index fa940d3f5..fd18a9caf 100644 --- a/twoliter/build.rs +++ b/twoliter/build.rs @@ -36,6 +36,7 @@ fn main() { paths.copy_file("repack.Dockerfile"); paths.copy_file("repack.Dockerfile.dockerignore"); paths.copy_file("rpm2img"); + paths.copy_file("rpm2kit"); paths.copy_file("rpm2kmodkit"); paths.copy_file("rpm2migrations"); paths.copy_file("metadata.spec"); diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index b432b1a8a..fa1523f85 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -750,6 +750,34 @@ cargo metadata \ ''' ] +# This task is temporarily necessary due to changes in Twoliter that occurred in June 2024. A new +# directory structure was introduced for build/rpms. Attempts to do an incremental build when the +# build output was in the old directory structure would fail, so we needed a way to inform users +# that they need to run a cargo make clean. +# +# Once the changed build/rpms structure has been around long enough (in, say, August 2024), this +# task can be removed as we can assume that no lingering old incremental builds exist in the wild. +# +# TODO - remove this task sometime around, or after, August 2024. +[tasks.check-rpm-structure] +script_runner = "bash" +script = [ +''' +if [ ! -d "${BUILDSYS_PACKAGES_DIR}" ]; then + exit 0 +fi + +count=$(find "${BUILDSYS_PACKAGES_DIR}" -maxdepth 1 -name "*.rpm" | wc -l) + +if [ "${count}" != "0" ]; then + echo "ERROR: You have rpms in a flat structure in ${BUILDSYS_PACKAGES_DIR}" + echo "Twoliter and Buildsys have been updated to use a different RPM directory structure" + echo "PLEASE RUN: cargo make clean" + exit 1 +fi +''' +] + # Builds a package including its build-time and runtime dependency packages. [tasks.build-package] dependencies = ["check-cargo-version", "fetch", "publish-setup", "fetch-licenses", "cargo-metadata"] diff --git a/twoliter/embedded/build.Dockerfile b/twoliter/embedded/build.Dockerfile index 3c619f797..d01a71ed6 100644 --- a/twoliter/embedded/build.Dockerfile +++ b/twoliter/embedded/build.Dockerfile @@ -1,7 +1,8 @@ # syntax=docker/dockerfile:1.4.3 -# This Dockerfile has two sections which are used to build rpm.spec packages and to create -# Bottlerocket images, respectively. They are marked as Section 1 and Section 2. buildsys -# uses Section 1 during build-package calls and Section 2 during build-variant calls. +# This Dockerfile has three sections which are used to build rpm.spec packages, to create +# kits, and to create Bottlerocket images, respectively. They are marked as Sections 1-3. +# buildsys uses Section 1 during build-package calls, Section 2 during build-kit calls, +# and Section 3 during build-variant calls. # # Several commands start with RUN --mount=target=/host, which mounts the docker build # context (which in practice is the root of the Bottlerocket repository) as a read-only @@ -94,6 +95,8 @@ RUN \ # Builds an RPM package from a spec file. FROM sdk AS rpmbuild ARG PACKAGE +ARG PACKAGE_DEPENDENCIES +ARG KIT_DEPENDENCIES ARG ARCH ARG NOCACHE ARG VARIANT @@ -129,18 +132,26 @@ RUN \ USER root RUN --mount=target=/host \ - ln -s /host/build/rpms/*.rpm ./rpmbuild/RPMS \ - && createrepo_c \ + for pkg in ${PACKAGE_DEPENDENCIES} ; do \ + ln -s /host/build/rpms/${pkg}/*.rpm ./rpmbuild/RPMS ; \ + done && \ + createrepo_c \ -o ./rpmbuild/RPMS \ -x '*-debuginfo-*.rpm' \ -x '*-debugsource-*.rpm' \ --no-database \ - /host/build/rpms \ - && cp .rpmmacros /etc/rpm/macros \ - && dnf -y \ + ./rpmbuild/RPMS && \ + cp .rpmmacros /etc/rpm/macros && \ + declare -a KIT_REPOS && \ + for kit in ${KIT_DEPENDENCIES} ; do \ + KIT_REPOS+=("--repofrompath=${kit},/host/build/kits/${kit}/${ARCH}" --enablerepo "${kit}") ; \ + done && \ + echo "${KIT_REPOS[@]}" && \ + dnf -y \ --disablerepo '*' \ --repofrompath repo,./rpmbuild/RPMS \ --enablerepo 'repo' \ + "${KIT_REPOS[@]}" \ --nogpgcheck \ --forcearch "${ARCH}" \ builddep rpmbuild/SPECS/${PACKAGE}.spec @@ -167,13 +178,47 @@ FROM scratch AS package COPY --from=rpmbuild /home/builder/rpmbuild/RPMS/*/*.rpm /output/ ############################################################################################ -# Section 2: The following build stages are used to create a Bottlerocket image once all of -# the rpm files have been created by repeatedly using Section 1. +# Section 2: The following build stages are used to create a Bottlerocket kit once all of +# the rpm files have been created by repeatedly using Section 1. This process can occur more +# than once because packages can depend on kits and those kits depend on packages that must +# be built first. # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -# Creates an RPM repository from packages created in Section 1. +# Builds a kit from RPM packages. +FROM sdk AS kitbuild +ARG KIT +ARG PACKAGE_DEPENDENCIES +ARG ARCH +ARG NOCACHE + +WORKDIR /home/builder +USER builder + +RUN --mount=target=/host \ + /host/build/tools/rpm2kit \ + --packages-dir=/host/build/rpms \ + --arch="${ARCH}" \ + "${PACKAGE_DEPENDENCIES[@]/#/--package=}" \ + --output-dir=/home/builder/output \ + && echo ${NOCACHE} + +# Copies kit artifacts from the previous stage to their expected location so that buildsys +# can find them and copy them out. +FROM scratch AS kit +COPY --from=kitbuild /home/builder/output/. /output/ + +############################################################################################ +# Section 3: The following build stages are used to create a Bottlerocket image once all of +# the rpm files have been created by repeatedly using Sections 1 and 2. + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# Creates an RPM repository from packages created in Section 1 and kits from Section 2. FROM sdk AS repobuild +# The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section. ARG PACKAGES +# The complete list of non-kit packages required by way of pure package-to-package dependencies. +ARG PACKAGE_DEPENDENCIES +ARG KIT_DEPENDENCIES ARG ARCH ARG NOCACHE @@ -196,28 +241,36 @@ RUN --mount=target=/host \ WORKDIR /root USER root RUN --mount=target=/host \ - mkdir -p /local/rpms ./rpmbuild/RPMS \ - && ln -s /host/build/rpms/*.rpm ./rpmbuild/RPMS \ - && ln -s /home/builder/rpmbuild/RPMS/*/*.rpm ./rpmbuild/RPMS \ - && createrepo_c \ + mkdir -p ./rpmbuild/RPMS && \ + for pkg in ${PACKAGE_DEPENDENCIES} ; do \ + ln -s /host/build/rpms/${pkg}/*.rpm ./rpmbuild/RPMS ; \ + done && \ + ln -s /home/builder/rpmbuild/RPMS/*/*.rpm ./rpmbuild/RPMS && \ + createrepo_c \ -o ./rpmbuild/RPMS \ -x '*-debuginfo-*.rpm' \ -x '*-debugsource-*.rpm' \ --no-database \ - ./rpmbuild/RPMS \ - && echo '%_dbpath %{_sharedstatedir}/rpm' >> /etc/rpm/macros \ - && dnf -y \ + ./rpmbuild/RPMS && \ + echo '%_dbpath %{_sharedstatedir}/rpm' >> /etc/rpm/macros && \ + declare -a KIT_REPOS && \ + for kit in ${KIT_DEPENDENCIES} ; do \ + KIT_REPOS+=("--repofrompath=${kit},/host/build/kits/${kit}/${ARCH}" --enablerepo "${kit}") ; \ + done && \ + dnf -y \ --disablerepo '*' \ --repofrompath repo,./rpmbuild/RPMS \ --enablerepo 'repo' \ + "${KIT_REPOS[@]}" \ --nogpgcheck \ --downloadonly \ --downloaddir . \ --forcearch "${ARCH}" \ - install $(printf "bottlerocket-%s\n" metadata ${PACKAGES}) \ - && mv *.rpm /local/rpms \ - && createrepo_c /local/rpms \ - && echo ${NOCACHE} + install $(printf "bottlerocket-%s\n" metadata ${PACKAGES}) && \ + mkdir -p /local/rpms && \ + mv *.rpm /local/rpms && \ + createrepo_c /local/rpms && \ + echo ${NOCACHE} # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # Builds a Bottlerocket image. @@ -288,7 +341,7 @@ WORKDIR /root USER root RUN --mount=target=/host \ mkdir -p /local/migrations \ - && find /host/build/rpms/ -maxdepth 1 -type f \ + && find /host/build/rpms/os/ -maxdepth 1 -type f \ -name "bottlerocket-migrations-*.rpm" \ -not -iname '*debuginfo*' \ -exec cp '{}' '/local/migrations/' ';' \ @@ -300,6 +353,7 @@ RUN --mount=target=/host \ # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # Creates an archive of kernel development sources and toolchain. FROM repobuild as kmodkitbuild +# The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section. ARG PACKAGES ARG ARCH ARG VERSION_ID @@ -314,7 +368,7 @@ WORKDIR /tmp RUN --mount=target=/host \ mkdir -p /local/archives \ && KERNEL="$(printf "%s\n" ${PACKAGES} | awk '/^kernel-/{print $1}')" \ - && find /host/build/rpms/ -maxdepth 1 -type f \ + && find /host/build/rpms/${KERNEL}/ -maxdepth 1 -type f \ -name "bottlerocket-${KERNEL}-archive-*.rpm" \ -exec cp '{}' '/local/archives/' ';' \ && /host/build/tools/rpm2kmodkit \ diff --git a/twoliter/embedded/build.Dockerfile.dockerignore b/twoliter/embedded/build.Dockerfile.dockerignore index c2bf40d23..b31d7ba42 100644 --- a/twoliter/embedded/build.Dockerfile.dockerignore +++ b/twoliter/embedded/build.Dockerfile.dockerignore @@ -6,6 +6,10 @@ !/build/rpms/*.rpm /build/rpms/*-debuginfo-*.rpm /build/rpms/*-debugsource-*.rpm +!/build/rpms/*/*.rpm +/build/rpms/*/*-debuginfo-*.rpm +/build/rpms/*/*-debugsource-*.rpm +!/build/kits/ !/build/tools/* **/target/* /sbkeys diff --git a/twoliter/embedded/rpm2kit b/twoliter/embedded/rpm2kit new file mode 100755 index 000000000..feaf7c476 --- /dev/null +++ b/twoliter/embedded/rpm2kit @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Create a kit from RPM package inputs. +set -eu -o pipefail + +declare -a PACKAGES + +for opt in "$@"; do + optarg="$(expr "${opt}" : '[^=]*=\(.*\)')" + case "${opt}" in + --packages-dir=*) PACKAGES_DIR="${optarg}" ;; + --package=*) PACKAGES+=("${optarg}") ;; + --output-dir=*) OUTPUT_DIR="${optarg}" ;; + esac +done + +KIT_DIR="${OUTPUT_DIR}/${ARCH}" +rm -rf "${KIT_DIR}" + +mkdir -p "${KIT_DIR}/Packages" +for pkg in ${PACKAGES} ; do + find "${PACKAGES_DIR}/${pkg}" \ + -mindepth 1 \ + -maxdepth 1 \ + ! -name '*-debuginfo-*' \ + ! -name '*-debugsource-*' \ + -size +0c \ + -exec install -p -m 0644 -t "${KIT_DIR}/Packages" {} \+ +done + +createrepo_c "${KIT_DIR}" +dnf --disablerepo '*' --repofrompath "kit,file:///${KIT_DIR}" repoquery --all diff --git a/twoliter/src/tools.rs b/twoliter/src/tools.rs index c80a17c42..ffe8c5443 100644 --- a/twoliter/src/tools.rs +++ b/twoliter/src/tools.rs @@ -115,6 +115,7 @@ async fn test_install_tools() { assert!(toolsdir.join("repack.Dockerfile").is_file()); assert!(toolsdir.join("repack.Dockerfile.dockerignore").is_file()); assert!(toolsdir.join("rpm2img").is_file()); + assert!(toolsdir.join("rpm2kit").is_file()); assert!(toolsdir.join("rpm2kmodkit").is_file()); assert!(toolsdir.join("rpm2migrations").is_file());