buildsys: add repack-variant target

Add the the initial target for repackaging a Bottlerocket variant. This
target will skip the traditional workflow of building packages and
using `rpm2img`. Instead the build system will point to a yet to be
implemented `img2img` tool.

Signed-off-by: Patrick J.P. Culp <jpculp@amazon.com>

Author: jpculp

docs/design/README.md

@@ -278,7 +278,7 @@ These will not be available and need to be added to the Bottlerocket SDK:
 The existing build system relies on certain binaries, scripts and static files, such as
 
 - buildsys, pubsys, testsys, etc.
-- rpm2img, the Dockerfile, and Makefile.toml
+- rpm2img, the Dockerfiles, and Makefile.toml
 
 These tools and files will be embedded into the Twoliter binary and  installed on-the-fly when Twoliter runs.
 The binaries will be built and embedded using the unstable Cargo feature [bindeps].

tools/buildsys/src/args.rs

@@ -41,13 +41,15 @@ pub(crate) struct Buildsys {
 pub(crate) enum Command {
     BuildPackage(Box<BuildPackageArgs>),
     BuildVariant(Box<BuildVariantArgs>),
+    RepackVariant(Box<RepackVariantArgs>),
 }
 
 impl Command {
     pub(crate) fn build_type(&self) -> BuildType {
         match self {
             Command::BuildPackage(_) => BuildType::Package,
             Command::BuildVariant(_) => BuildType::Variant,
+            Command::RepackVariant(_) => BuildType::Repack,
         }
     }
 }
@@ -160,6 +162,25 @@ pub(crate) struct BuildVariantArgs {
     pub(crate) common: Common,
 }
 
+/// Repack variant from prebuilt images.
+#[derive(Debug, Parser)]
+pub(crate) struct RepackVariantArgs {
+    #[arg(long, env = "BUILDSYS_NAME")]
+    pub(crate) name: String,
+
+    #[arg(long, env = "BUILDSYS_VARIANT")]
+    pub(crate) variant: String,
+
+    #[arg(long, env = "BUILDSYS_VERSION_BUILD")]
+    pub(crate) version_build: String,
+
+    #[arg(long, env = "BUILDSYS_VERSION_IMAGE")]
+    pub(crate) version_image: String,
+
+    #[command(flatten)]
+    pub(crate) common: Common,
+}
+
 /// Returns the environment variables that need to be watched for a given `[BuildType]`.
 fn sensitive_env_vars(build_type: BuildFlags) -> impl Iterator<Item = &'static str> {
     REBUILD_VARS
@@ -183,6 +204,7 @@ pub(crate) fn rerun_for_envs(build_type: BuildType) {
 #[repr(u8)]
 #[derive(Debug, Clone, Copy)]
 enum BuildFlags {
+    Repack = 0b00000000,
     Package = 0b00000001,
     Kit = 0b00000010,
     Variant = 0b00000100,
@@ -191,6 +213,7 @@ enum BuildFlags {
 impl From<BuildType> for BuildFlags {
     fn from(value: BuildType) -> Self {
         match value {
+            BuildType::Repack => BuildFlags::Repack,
             BuildType::Package => BuildFlags::Package,
             BuildType::Kit => BuildFlags::Kit,
             BuildType::Variant => BuildFlags::Variant,
@@ -212,11 +235,14 @@ const VARIANT: u8 = BuildFlags::Variant as u8;
 #[test]
 fn build_type_includes_test() {
     // true
+    assert!(BuildFlags::Repack.includes(0));
     assert!(BuildFlags::Package.includes(PACKAGE | VARIANT));
     assert!(BuildFlags::Variant.includes(VARIANT));
     assert!(BuildFlags::Variant.includes(VARIANT | PACKAGE));
 
     // false
+    assert!(!BuildFlags::Repack.includes(PACKAGE));
+    assert!(!BuildFlags::Repack.includes(VARIANT));
     assert!(!BuildFlags::Package.includes(VARIANT));
     assert!(!BuildFlags::Variant.includes(PACKAGE));
     assert!(!BuildFlags::Variant.includes(32));

tools/buildsys/src/builder.rs

@@ -6,7 +6,7 @@ the repository's top-level Dockerfile.
 */
 pub(crate) mod error;
 
-use crate::args::{BuildPackageArgs, BuildVariantArgs};
+use crate::args::{BuildPackageArgs, BuildVariantArgs, RepackVariantArgs};
 use buildsys::manifest::{
     ImageFeature, ImageFormat, ImageLayout, Manifest, PartitionPlan, SupportedArch,
 };
@@ -86,7 +86,6 @@ lazy_static! {
 
 static DOCKER_BUILD_MAX_ATTEMPTS: NonZeroU16 = nonzero!(10u16);
 
-#[allow(dead_code)]
 enum OutputCleanup {
     BeforeBuild,
     None,
@@ -224,17 +223,56 @@ impl VariantBuildArgs {
     }
 }
 
+struct VariantRepackArgs {
+    data_image_publish_size_gib: i32,
+    image_features: HashSet<ImageFeature>,
+    image_format: String,
+    name: String,
+    os_image_publish_size_gib: String,
+    partition_plan: String,
+    variant: String,
+    version_build: String,
+    version_image: String,
+}
+
+impl VariantRepackArgs {
+    fn build_args(&self) -> Vec<String> {
+        let mut args = Vec::new();
+        args.push("--network".into());
+        args.push("host".into());
+        args.build_arg(
+            "DATA_IMAGE_PUBLISH_SIZE_GIB",
+            self.data_image_publish_size_gib.to_string(),
+        );
+        args.build_arg("IMAGE_FORMAT", &self.image_format);
+        args.build_arg("IMAGE_NAME", &self.name);
+        args.build_arg("OS_IMAGE_PUBLISH_SIZE_GIB", &self.os_image_publish_size_gib);
+        args.build_arg("PARTITION_PLAN", &self.partition_plan);
+        args.build_arg("VARIANT", &self.variant);
+        args.build_arg("BUILD_ID", &self.version_build);
+        args.build_arg("VERSION_ID", &self.version_image);
+
+        for image_feature in self.image_features.iter() {
+            args.build_arg(format!("{}", image_feature), "1");
+        }
+
+        args
+    }
+}
+
 #[allow(clippy::large_enum_variant)]
 enum TargetBuildArgs {
     Package(PackageBuildArgs),
     Variant(VariantBuildArgs),
+    Repack(VariantRepackArgs),
 }
 
 impl TargetBuildArgs {
     pub(crate) fn build_type(&self) -> BuildType {
         match self {
             TargetBuildArgs::Package(_) => BuildType::Package,
             TargetBuildArgs::Variant(_) => BuildType::Variant,
+            TargetBuildArgs::Repack(_) => BuildType::Repack,
         }
     }
 }
@@ -384,6 +422,60 @@ impl DockerBuild {
         })
     }
 
+    /// Create a new `DockerBuild` that can repackage a variant image.
+    pub(crate) fn repack_variant(args: RepackVariantArgs, manifest: &Manifest) -> Result<Self> {
+        let image_layout = manifest.info().image_layout().cloned().unwrap_or_default();
+        let ImageLayout { partition_plan, .. } = image_layout;
+
+        let (os_image_publish_size_gib, data_image_publish_size_gib) =
+            image_layout.publish_image_sizes_gib();
+
+        Ok(Self {
+            dockerfile: args.common.tools_dir.join("repack.Dockerfile"),
+            context: args.common.root_dir.clone(),
+            target: "repack".to_string(),
+            tag: append_token(
+                format!(
+                    "buildsys-repack-{variant}-{arch}",
+                    variant = args.variant,
+                    arch = args.common.arch
+                ),
+                &args.common.root_dir,
+            ),
+            root_dir: args.common.root_dir.clone(),
+            artifacts_dir: args.common.image_arch_variant_dir,
+            state_dir: args.common.state_dir,
+            artifact_name: args.variant.clone(),
+            common_build_args: CommonBuildArgs::new(
+                &args.common.root_dir,
+                args.common.sdk_image,
+                args.common.arch,
+                OutputCleanup::None,
+            ),
+            target_build_args: TargetBuildArgs::Repack(VariantRepackArgs {
+                data_image_publish_size_gib,
+                image_features: manifest.info().image_features().unwrap_or_default(),
+                image_format: match manifest.info().image_format() {
+                    Some(ImageFormat::Raw) | None => "raw",
+                    Some(ImageFormat::Qcow2) => "qcow2",
+                    Some(ImageFormat::Vmdk) => "vmdk",
+                }
+                .to_string(),
+                name: args.name,
+                os_image_publish_size_gib: os_image_publish_size_gib.to_string(),
+                partition_plan: match partition_plan {
+                    PartitionPlan::Split => "split",
+                    PartitionPlan::Unified => "unified",
+                }
+                .to_string(),
+                variant: args.variant,
+                version_build: args.version_build,
+                version_image: args.version_image,
+            }),
+            secrets_args: secrets_args()?,
+        })
+    }
+
     pub(crate) fn build(&self) -> Result<()> {
         env::set_current_dir(&self.root_dir).context(error::DirectoryChangeSnafu {
             path: &self.root_dir,
@@ -468,6 +560,7 @@ impl DockerBuild {
         let mut args = match &self.target_build_args {
             TargetBuildArgs::Package(p) => p.build_args(),
             TargetBuildArgs::Variant(v) => v.build_args(),
+            TargetBuildArgs::Repack(r) => r.build_args(),
         };
         args.build_arg("ARCH", self.common_build_args.arch.to_string());
         args.build_arg("GOARCH", self.common_build_args.arch.goarch());
@@ -571,6 +664,7 @@ fn create_marker_dir(
         BuildType::Package => "packages",
         BuildType::Kit => "kits",
         BuildType::Variant => "variants",
+        BuildType::Repack => "repack",
     };
 
     let path = [&state_dir.display().to_string(), arch, prefix, name]

tools/buildsys/src/lib.rs

@@ -6,4 +6,5 @@ pub enum BuildType {
     Package,
     Kit,
     Variant,
+    Repack,
 }

tools/buildsys/src/main.rs

@@ -15,7 +15,7 @@ mod gomod;
 mod project;
 mod spec;
 
-use crate::args::{BuildPackageArgs, BuildVariantArgs, Buildsys, Command};
+use crate::args::{BuildPackageArgs, BuildVariantArgs, Buildsys, Command, RepackVariantArgs};
 use crate::builder::DockerBuild;
 use buildsys::manifest::{BundleModule, Manifest, ManifestInfo, SupportedArch};
 use cache::LookasideCache;
@@ -103,6 +103,7 @@ fn run(args: Buildsys) -> Result<()> {
     match args.command {
         Command::BuildPackage(args) => build_package(*args),
         Command::BuildVariant(args) => build_variant(*args),
+        Command::RepackVariant(args) => repack_variant(*args),
     }
 }
 
@@ -256,6 +257,23 @@ fn build_variant(args: BuildVariantArgs) -> Result<()> {
         .context(error::BuildAttemptSnafu)
 }
 
+fn repack_variant(args: RepackVariantArgs) -> Result<()> {
+    let manifest_file = "Cargo.toml";
+
+    let manifest = Manifest::new(
+        args.common.cargo_manifest_dir.join(manifest_file),
+        &args.common.cargo_metadata_path,
+    )
+    .context(error::ManifestParseSnafu)?;
+
+    supported_arch(manifest.info(), args.common.arch)?;
+
+    DockerBuild::repack_variant(args, &manifest)
+        .context(error::BuilderInstantiationSnafu)?
+        .build()
+        .context(error::BuildAttemptSnafu)
+}
+
 /// Ensure that the current arch is supported by the current variant
 fn supported_arch(manifest: &ManifestInfo, arch: SupportedArch) -> Result<()> {
     if let Some(supported_arches) = manifest.supported_arches() {

tools/buildsys/src/manifest.rs

@@ -460,6 +460,8 @@ impl ManifestInfo {
             Ok(BuildType::Kit)
         } else if self.build_variant().is_some() {
             Ok(BuildType::Variant)
+        } else if self.repack_variant().is_some() {
+            Ok(BuildType::Repack)
         } else {
             Err(Error(error::UnknownManifestTypeSnafu {}.build()))
         }
@@ -486,6 +488,13 @@ impl ManifestInfo {
             .as_ref()
             .and_then(|m| m.build_variant.as_ref())
     }
+
+    fn repack_variant(&self) -> Option<&RepackVariant> {
+        self.package
+            .metadata
+            .as_ref()
+            .and_then(|m| m.repack_variant.as_ref())
+    }
 }
 
 /// For the "top-level manifest", i.e. the thing that `buildsys` is building, only
@@ -504,6 +513,7 @@ fn is_manifest_type(pkg_metadata: &PackageMetadata, manifest_type: BuildType) ->
         BuildType::Package => metadata_table.get("build-package").is_some(),
         BuildType::Kit => metadata_table.get("build-kit").is_some(),
         BuildType::Variant => metadata_table.get("build-variant").is_some(),
+        BuildType::Repack => metadata_table.get("repack-variant").is_some(),
     }
 }
 
@@ -556,6 +566,7 @@ struct Metadata {
     build_package: Option<BuildPackage>,
     build_kit: Option<BuildKit>,
     build_variant: Option<BuildVariant>,
+    repack_variant: Option<RepackVariant>,
 }
 
 #[derive(Deserialize, Debug)]
@@ -606,6 +617,16 @@ pub struct BuildVariant {
     pub image_features: Option<HashMap<ImageFeature, bool>>,
 }
 
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct RepackVariant {
+    pub image_format: Option<ImageFormat>,
+    #[serde(default)]
+    pub image_layout: ImageLayout,
+    pub supported_arches: Option<HashSet<SupportedArch>>,
+    pub image_features: Option<HashMap<ImageFeature, bool>>,
+}
+
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "lowercase")]
 pub enum ImageFormat {

twoliter/build.rs

@@ -30,7 +30,10 @@ fn main() {
     paths.copy_file("build.Dockerfile");
     paths.copy_file("build.Dockerfile.dockerignore");
     paths.copy_file("docker-go");
+    paths.copy_file("img2img");
     paths.copy_file("partyplanner");
+    paths.copy_file("repack.Dockerfile");
+    paths.copy_file("repack.Dockerfile.dockerignore");
     paths.copy_file("rpm2img");
     paths.copy_file("rpm2kmodkit");
     paths.copy_file("rpm2migrations");

twoliter/embedded/Makefile.toml

@@ -811,6 +811,20 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${BUILDSYS_OUTPUT_DIR}/latest"
 '''
 ]
 
+[tasks.repack-variant]
+dependencies = ["fetch-sdk", "build-sbkeys", "publish-setup"]
+script = [
+'''
+export PATH="${TWOLITER_TOOLS_DIR}:${PATH}"
+
+# Parse the variant into its components and set additional variables.
+eval "$(bottlerocket-variant)"
+
+buildsys repack-variant \
+  --cargo-manifest-dir variants/${BUILDSYS_VARIANT}
+'''
+]
+
 [tasks.check-licenses]
 dependencies = ["fetch"]
 script = [

twoliter/embedded/img2img

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "img2img not yet implemented" >&2 ; exit 1