Merge pull request #227 from bottlerocket-os/build-kit

buildsys build-kit

Author: webern

tools/buildsys/src/args.rs

@@ -15,20 +15,21 @@ use url::Url;
 /// variable changes. The build type is represented with bit flags so that we can easily list
 /// multiple build types for a single variable. See `[BuildType]` and `[rerun_for_envs]` below to
 /// see how this list is used.
-const REBUILD_VARS: [(&str, u8); 13] = [
-    ("BUILDSYS_ARCH", PACKAGE | VARIANT),
-    ("BUILDSYS_OUTPUT_GENERATION_ID", PACKAGE | VARIANT),
+const REBUILD_VARS: [(&str, u8); 14] = [
+    ("BUILDSYS_ARCH", PACKAGE | KIT | VARIANT),
+    ("BUILDSYS_KITS_DIR", KIT),
     ("BUILDSYS_NAME", VARIANT),
     ("BUILDSYS_OUTPUT_DIR", VARIANT),
+    ("BUILDSYS_OUTPUT_GENERATION_ID", PACKAGE | KIT | VARIANT),
     ("BUILDSYS_PACKAGES_DIR", PACKAGE),
     ("BUILDSYS_PRETTY_NAME", VARIANT),
-    ("BUILDSYS_ROOT_DIR", PACKAGE | VARIANT),
-    ("BUILDSYS_STATE_DIR", PACKAGE | VARIANT),
+    ("BUILDSYS_ROOT_DIR", PACKAGE | KIT | VARIANT),
+    ("BUILDSYS_STATE_DIR", PACKAGE | KIT | VARIANT),
     ("BUILDSYS_TIMESTAMP", VARIANT),
     ("BUILDSYS_VARIANT", VARIANT),
     ("BUILDSYS_VERSION_BUILD", VARIANT),
     ("BUILDSYS_VERSION_IMAGE", VARIANT),
-    ("TLPRIVATE_SDK_IMAGE", PACKAGE | VARIANT),
+    ("TLPRIVATE_SDK_IMAGE", PACKAGE | KIT | VARIANT),
 ];
 
 /// A tool for building Bottlerocket images and artifacts.
@@ -41,6 +42,7 @@ pub(crate) struct Buildsys {
 #[derive(Subcommand, Debug)]
 pub(crate) enum Command {
     BuildPackage(Box<BuildPackageArgs>),
+    BuildKit(Box<BuildKitArgs>),
     BuildVariant(Box<BuildVariantArgs>),
     RepackVariant(Box<RepackVariantArgs>),
 }
@@ -49,6 +51,7 @@ impl Command {
     pub(crate) fn build_type(&self) -> BuildType {
         match self {
             Command::BuildPackage(_) => BuildType::Package,
+            Command::BuildKit(_) => BuildType::Kit,
             Command::BuildVariant(_) => BuildType::Variant,
             Command::RepackVariant(_) => BuildType::Repack,
         }
@@ -126,6 +129,21 @@ pub(crate) struct BuildPackageArgs {
     pub(crate) common: Common,
 }
 
+/// Place the required RPMs into a kit (directory) and make a yum repo.
+#[derive(Debug, Parser)]
+pub(crate) struct BuildKitArgs {
+    /// The directory where built RPMs go, e.g. build/rpms
+    #[arg(long, env = "BUILDSYS_PACKAGES_DIR")]
+    pub(crate) packages_dir: PathBuf,
+
+    /// The directory where built kits go, e.g. build/kits
+    #[arg(long, env = "BUILDSYS_KITS_DIR")]
+    pub(crate) kits_dir: PathBuf,
+
+    #[command(flatten)]
+    pub(crate) common: Common,
+}
+
 /// Build filesystem and disk images from RPMs.
 #[derive(Debug, Parser)]
 pub(crate) struct BuildVariantArgs {
@@ -230,6 +248,7 @@ impl BuildFlags {
 #[allow(dead_code)]
 const REPACK: u8 = BuildFlags::Repack as u8;
 const PACKAGE: u8 = BuildFlags::Package as u8;
+const KIT: u8 = BuildFlags::Kit as u8;
 const VARIANT: u8 = BuildFlags::Variant as u8;
 
 #[test]

tools/buildsys/src/builder.rs

@@ -6,7 +6,7 @@ the repository's top-level Dockerfile.
 */
 pub(crate) mod error;
 
-use crate::args::{BuildPackageArgs, BuildVariantArgs, RepackVariantArgs};
+use crate::args::{BuildKitArgs, BuildPackageArgs, BuildVariantArgs, RepackVariantArgs};
 use buildsys::manifest::{
     ImageFeature, ImageFormat, ImageLayout, Manifest, PartitionPlan, SupportedArch,
 };
@@ -142,7 +142,23 @@ struct PackageBuildArgs {
     variant_runtime: String,
 }
 
-impl PackageBuildArgs {
+impl KitBuildArgs {
+    fn build_args(&self) -> Vec<String> {
+        let mut args = Vec::new();
+        args.push("--network".into());
+        args.push("none".into());
+        args.build_arg("KIT", &self.kit);
+        args.build_arg("PACKAGE_DEPENDENCIES", self.package_dependencies.join(" "));
+        args
+    }
+}
+
+struct KitBuildArgs {
+    kit: String,
+    package_dependencies: Vec<String>,
+}
+
+impl crate::builder::PackageBuildArgs {
     fn build_args(&self) -> Vec<String> {
         let mut args = Vec::new();
         args.push("--network".into());
@@ -267,6 +283,7 @@ impl RepackVariantBuildArgs {
 #[allow(clippy::large_enum_variant)]
 enum TargetBuildArgs {
     Package(PackageBuildArgs),
+    Kit(KitBuildArgs),
     Variant(VariantBuildArgs),
     Repack(RepackVariantBuildArgs),
 }
@@ -275,6 +292,7 @@ impl TargetBuildArgs {
     pub(crate) fn build_type(&self) -> BuildType {
         match self {
             TargetBuildArgs::Package(_) => BuildType::Package,
+            TargetBuildArgs::Kit(_) => BuildType::Kit,
             TargetBuildArgs::Variant(_) => BuildType::Variant,
             TargetBuildArgs::Repack(_) => BuildType::Repack,
         }
@@ -343,6 +361,40 @@ impl DockerBuild {
         })
     }
 
+    pub(crate) fn new_kit(args: BuildKitArgs, manifest: &Manifest) -> Result<Self> {
+        let kit = manifest.info().kit_name();
+        let per_kit_dir = format!("{}/{}", args.kits_dir.display(), kit).into();
+
+        Ok(Self {
+            dockerfile: args.common.tools_dir.join("build.Dockerfile"),
+            context: args.common.root_dir.clone(),
+            target: "kit".to_string(),
+            tag: append_token(
+                format!(
+                    "buildsys-kit-{kit}-{arch}",
+                    kit = kit,
+                    arch = args.common.arch,
+                ),
+                &args.common.root_dir,
+            ),
+            root_dir: args.common.root_dir.clone(),
+            artifacts_dir: per_kit_dir,
+            state_dir: args.common.state_dir,
+            artifact_name: kit.to_string(),
+            common_build_args: CommonBuildArgs::new(
+                &args.common.root_dir,
+                args.common.sdk_image,
+                args.common.arch,
+                OutputCleanup::BeforeBuild,
+            ),
+            target_build_args: TargetBuildArgs::Kit(KitBuildArgs {
+                kit: kit.to_string(),
+                package_dependencies: manifest.package_dependencies().context(error::GraphSnafu)?,
+            }),
+            secrets_args: Vec::new(),
+        })
+    }
+
     /// Create a new `DockerBuild` that can build a variant image.
     pub(crate) fn new_variant(args: BuildVariantArgs, manifest: &Manifest) -> Result<Self> {
         let image_layout = manifest.info().image_layout().cloned().unwrap_or_default();
@@ -567,6 +619,7 @@ impl DockerBuild {
     fn build_args(&self) -> Vec<String> {
         let mut args = match &self.target_build_args {
             TargetBuildArgs::Package(p) => p.build_args(),
+            TargetBuildArgs::Kit(k) => k.build_args(),
             TargetBuildArgs::Variant(v) => v.build_args(),
             TargetBuildArgs::Repack(r) => r.build_args(),
         };

tools/buildsys/src/main.rs

@@ -15,7 +15,9 @@ mod gomod;
 mod project;
 mod spec;
 
-use crate::args::{BuildPackageArgs, BuildVariantArgs, Buildsys, Command, RepackVariantArgs};
+use crate::args::{
+    BuildKitArgs, BuildPackageArgs, BuildVariantArgs, Buildsys, Command, RepackVariantArgs,
+};
 use crate::builder::DockerBuild;
 use buildsys::manifest::{BundleModule, ImageFeature, Manifest, ManifestInfo, SupportedArch};
 use cache::LookasideCache;
@@ -112,6 +114,7 @@ fn run(args: Buildsys) -> Result<()> {
     args::rerun_for_envs(args.command.build_type());
     match args.command {
         Command::BuildPackage(args) => build_package(*args),
+        Command::BuildKit(args) => build_kit(*args),
         Command::BuildVariant(args) => build_variant(*args),
         Command::RepackVariant(args) => repack_variant(*args),
     }
@@ -199,6 +202,22 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
         .context(error::BuildAttemptSnafu)
 }
 
+fn build_kit(args: BuildKitArgs) -> Result<()> {
+    let manifest_file = "Cargo.toml";
+    println!("cargo:rerun-if-changed={}", manifest_file);
+
+    let manifest = Manifest::new(
+        args.common.cargo_manifest_dir.join(manifest_file),
+        &args.common.cargo_metadata_path,
+    )
+    .context(error::ManifestParseSnafu)?;
+
+    DockerBuild::new_kit(args, &manifest)
+        .context(error::BuilderInstantiationSnafu)?
+        .build()
+        .context(error::BuildAttemptSnafu)
+}
+
 fn build_variant(args: BuildVariantArgs) -> Result<()> {
     let manifest_file = "Cargo.toml";
     println!("cargo:rerun-if-changed={}", manifest_file);

tools/buildsys/src/manifest.rs

@@ -369,6 +369,15 @@ impl ManifestInfo {
             .unwrap_or_else(|| self.manifest_name())
     }
 
+    /// Convenience method to return the kit name. If the manifest has an override in the
+    /// `package.metadata.build-kit.kit-name` key, it is returned, otherwise the Cargo manifest name
+    /// is returned from `package.name`.
+    pub fn kit_name(&self) -> &str {
+        self.build_kit()
+            .and_then(|b| b.kit_name.as_deref())
+            .unwrap_or_else(|| self.manifest_name())
+    }
+
     /// Convenience method to find whether the package is sensitive to variant changes.
     pub fn variant_sensitive(&self) -> Option<&VariantSensitivity> {
         self.build_package()
@@ -542,7 +551,7 @@ pub struct BuildPackage {
 #[serde(rename_all = "kebab-case")]
 #[allow(dead_code)]
 pub struct BuildKit {
-    pub included_packages: Option<Vec<String>>,
+    pub kit_name: Option<String>,
 }
 
 #[derive(Deserialize, Debug)]

twoliter/embedded/Makefile.toml

@@ -11,6 +11,7 @@ BUILDSYS_ARCH = { script = ['echo "${BUILDSYS_ARCH:-$(uname -m)}"'] }
 BUILDSYS_ROOT_DIR = "${CARGO_MAKE_WORKING_DIRECTORY}"
 BUILDSYS_BUILD_DIR = "${BUILDSYS_ROOT_DIR}/build"
 BUILDSYS_PACKAGES_DIR = "${BUILDSYS_BUILD_DIR}/rpms"
+BUILDSYS_KITS_DIR = "${BUILDSYS_BUILD_DIR}/kits"
 BUILDSYS_STATE_DIR = "${BUILDSYS_BUILD_DIR}/state"
 BUILDSYS_IMAGES_DIR = "${BUILDSYS_BUILD_DIR}/images"
 BUILDSYS_TOOLS_DIR = "${BUILDSYS_ROOT_DIR}/tools"
@@ -280,6 +281,7 @@ fi
 mkdir -p ${BUILDSYS_BUILD_DIR}
 mkdir -p ${BUILDSYS_OUTPUT_DIR}
 mkdir -p ${BUILDSYS_PACKAGES_DIR}
+mkdir -p ${BUILDSYS_KITS_DIR}
 mkdir -p ${BUILDSYS_STATE_DIR}
 mkdir -p ${BUILDSYS_METADATA_DIR}
 mkdir -p ${GO_MOD_CACHE}
@@ -1531,11 +1533,13 @@ extend = "_upload-ova-base"
 dependencies = [
   "clean-sources",
   "clean-packages",
+  "clean-kits",
   "clean-images",
   "clean-repos",
   "clean-state",
   "clean-tools",
   "clean-metadata",
+  "clean-workspace",
 ]
 
 [tasks.clean-sources]
@@ -1551,12 +1555,27 @@ rm -f ${BUILDSYS_TOOLS_DIR}/bin/*
 '''
 ]
 
+[tasks.clean-workspace]
+script_runner = "bash"
+script = [
+'''
+cargo clean --manifest-path "${BUILDSYS_ROOT_DIR}/Cargo.toml"
+'''
+]
+
 [tasks.clean-packages]
 script_runner = "bash"
 script = [
 '''
 rm -rf ${BUILDSYS_PACKAGES_DIR}
-cargo clean --manifest-path "${BUILDSYS_ROOT_DIR}/Cargo.toml"
+'''
+]
+
+[tasks.clean-kits]
+script_runner = "bash"
+script = [
+'''
+rm -rf ${BUILDSYS_KITS_DIR}
 '''
 ]