From 78874585569ad36e86d2f29de124d59c90e40a5a Mon Sep 17 00:00:00 2001 From: "Patrick J.P. Culp" Date: Tue, 14 May 2024 00:50:43 +0000 Subject: [PATCH 1/2] buildsys: preliminary housekeeping Adjusts some things in the build system for preliminary support for an additional build target. Removes some unnecessary checks, references, and dependencies. Renames Dockerfile to build.Dockerfile and adds a common argument for future build targets to skip cleanup if desired. Signed-off-by: Patrick J.P. Culp --- docs/design/README.md | 4 +-- tools/buildsys/src/builder.rs | 30 +++++++++++++++---- tools/buildsys/src/main.rs | 16 ++++------ twoliter/build.rs | 4 +-- twoliter/embedded/Makefile.toml | 4 +-- .../embedded/{Dockerfile => build.Dockerfile} | 0 ...erignore => build.Dockerfile.dockerignore} | 0 twoliter/src/tools.rs | 11 ++++--- 8 files changed, 43 insertions(+), 26 deletions(-) rename twoliter/embedded/{Dockerfile => build.Dockerfile} (100%) rename twoliter/embedded/{Dockerfile.dockerignore => build.Dockerfile.dockerignore} (100%) diff --git a/docs/design/README.md b/docs/design/README.md index fbfe107b6..fdd486f62 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -278,7 +278,7 @@ These will not be available and need to be added to the Bottlerocket SDK: The existing build system relies on certain binaries, scripts and static files, such as - buildsys, pubsys, testsys, etc. -- rpm2img, Dockerfile, and Makefile.toml +- rpm2img, the Dockerfile, and Makefile.toml These tools and files will be embedded into the Twoliter binary and installed on-the-fly when Twoliter runs. The binaries will be built and embedded using the unstable Cargo feature [bindeps]. @@ -292,7 +292,7 @@ We can pin the project to a specific toolchain using `rust-toolchain.toml` and w Certain scripts used during the Bottlerocket build system are tightly coupled to the operation of twoliter. These will be moved out of the Bottlerocket monorepo and into the twoliter git repository. Before using the SDK, these scripts will need to be copied into it. -These will be copied as part of the build system's main Dockerfile. +These will be copied as part of the build system's main Dockerfile (`build.Dockerfile`). ### Docker Domain Socket diff --git a/tools/buildsys/src/builder.rs b/tools/buildsys/src/builder.rs index de42d5614..707ea6960 100644 --- a/tools/buildsys/src/builder.rs +++ b/tools/buildsys/src/builder.rs @@ -86,15 +86,27 @@ lazy_static! { static DOCKER_BUILD_MAX_ATTEMPTS: NonZeroU16 = nonzero!(10u16); +#[allow(dead_code)] +enum OutputCleanup { + BeforeBuild, + None, +} + struct CommonBuildArgs { arch: SupportedArch, sdk: String, nocache: String, token: String, + cleanup: OutputCleanup, } impl CommonBuildArgs { - fn new(root: impl AsRef, sdk: String, arch: SupportedArch) -> Self { + fn new( + root: impl AsRef, + sdk: String, + arch: SupportedArch, + cleanup: OutputCleanup, + ) -> Self { let mut d = Sha512::new(); d.update(root.as_ref().display().to_string()); let digest = hex::encode(d.finalize()); @@ -108,6 +120,7 @@ impl CommonBuildArgs { sdk, nocache, token, + cleanup, } } } @@ -254,7 +267,7 @@ impl DockerBuild { }; Ok(Self { - dockerfile: args.common.tools_dir.join("Dockerfile"), + dockerfile: args.common.tools_dir.join("build.Dockerfile"), context: args.common.root_dir.clone(), target: "package".to_string(), tag: append_token( @@ -273,6 +286,7 @@ impl DockerBuild { &args.common.root_dir, args.common.sdk_image, args.common.arch, + OutputCleanup::BeforeBuild, ), target_build_args: TargetBuildArgs::Package(PackageBuildArgs { image_features, @@ -304,7 +318,7 @@ impl DockerBuild { image_layout.publish_image_sizes_gib(); Ok(Self { - dockerfile: args.common.tools_dir.join("Dockerfile"), + dockerfile: args.common.tools_dir.join("build.Dockerfile"), context: args.common.root_dir.clone(), target: "variant".to_string(), tag: append_token( @@ -323,6 +337,7 @@ impl DockerBuild { &args.common.root_dir, args.common.sdk_image, args.common.arch, + OutputCleanup::BeforeBuild, ), target_build_args: TargetBuildArgs::Variant(VariantBuildArgs { package_dependencies: manifest.package_dependencies().context(error::GraphSnafu)?, @@ -383,7 +398,12 @@ impl DockerBuild { )?; // Clean up any previous outputs we have tracked. - clean_build_files(&marker_dir, &self.artifacts_dir)?; + match self.common_build_args.cleanup { + OutputCleanup::BeforeBuild => { + clean_build_files(&marker_dir, &self.artifacts_dir)?; + } + OutputCleanup::None => (), + } let mut build = format!( "build {context} \ @@ -526,7 +546,7 @@ fn secrets_args() -> Result> { ); } - for var in &[ + for var in [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", diff --git a/tools/buildsys/src/main.rs b/tools/buildsys/src/main.rs index 320d62766..318e0e326 100644 --- a/tools/buildsys/src/main.rs +++ b/tools/buildsys/src/main.rs @@ -235,8 +235,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> { DockerBuild::new_package(args, &manifest, image_features.unwrap_or_default()) .context(error::BuilderInstantiationSnafu)? .build() - .context(error::BuildAttemptSnafu)?; - Ok(()) + .context(error::BuildAttemptSnafu) } fn build_variant(args: BuildVariantArgs) -> Result<()> { @@ -251,15 +250,10 @@ fn build_variant(args: BuildVariantArgs) -> Result<()> { supported_arch(manifest.info(), args.common.arch)?; - if manifest.info().included_packages().is_some() { - DockerBuild::new_variant(args, &manifest) - .context(error::BuilderInstantiationSnafu)? - .build() - .context(error::BuildAttemptSnafu)?; - } else { - println!("cargo:warning=No included packages in manifest. Skipping variant build."); - } - Ok(()) + DockerBuild::new_variant(args, &manifest) + .context(error::BuilderInstantiationSnafu)? + .build() + .context(error::BuildAttemptSnafu) } /// Ensure that the current arch is supported by the current variant diff --git a/twoliter/build.rs b/twoliter/build.rs index b097daad6..86d736982 100644 --- a/twoliter/build.rs +++ b/twoliter/build.rs @@ -26,9 +26,9 @@ fn main() { paths.prep_dir.display() )); - paths.copy_file("Dockerfile"); - paths.copy_file("Dockerfile.dockerignore"); paths.copy_file("Makefile.toml"); + paths.copy_file("build.Dockerfile"); + paths.copy_file("build.Dockerfile.dockerignore"); paths.copy_file("docker-go"); paths.copy_file("imghelper"); paths.copy_file("partyplanner"); diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index 82b73c04a..832014459 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -611,7 +611,7 @@ fi ] [tasks.build-sbkeys] -dependencies = ["fetch"] +dependencies = ["fetch-sdk"] script_runner = "bash" script = [ ''' @@ -1672,7 +1672,7 @@ script = [ # This task will call watch on the `status` testsys command to show the results of all tests and # resources. -# To see all incomplete crds use `cargo make watch-test-all --running` +# To see all incomplete crds use `cargo make watch-test-all --running` [tasks.watch-test-all] script = [ ''' diff --git a/twoliter/embedded/Dockerfile b/twoliter/embedded/build.Dockerfile similarity index 100% rename from twoliter/embedded/Dockerfile rename to twoliter/embedded/build.Dockerfile diff --git a/twoliter/embedded/Dockerfile.dockerignore b/twoliter/embedded/build.Dockerfile.dockerignore similarity index 100% rename from twoliter/embedded/Dockerfile.dockerignore rename to twoliter/embedded/build.Dockerfile.dockerignore diff --git a/twoliter/src/tools.rs b/twoliter/src/tools.rs index 8b75514f0..1cf2ce62f 100644 --- a/twoliter/src/tools.rs +++ b/twoliter/src/tools.rs @@ -37,7 +37,7 @@ pub(crate) async fn install_tools(tools_dir: impl AsRef) -> Result<()> { .context("Unable to install tools")?; // Pick one of the embedded files for use as the canonical mtime. - let metadata = fs::metadata(dir.join("Dockerfile")) + let metadata = fs::metadata(dir.join("build.Dockerfile")) .await .context("Unable to get Dockerfile metadata")?; let mtime = FileTime::from_last_modification_time(&metadata); @@ -104,15 +104,16 @@ async fn test_install_tools() { // Assert that the expected files exist in the tools directory. // Check that non-binary files were copied. - assert!(toolsdir.join("Dockerfile").is_file()); assert!(toolsdir.join("Makefile.toml").is_file()); + assert!(toolsdir.join("build.Dockerfile").is_file()); + assert!(toolsdir.join("build.Dockerfile.dockerignore").is_file()); assert!(toolsdir.join("docker-go").is_file()); assert!(toolsdir.join("imghelper").is_file()); + assert!(toolsdir.join("metadata.spec").is_file()); assert!(toolsdir.join("partyplanner").is_file()); assert!(toolsdir.join("rpm2img").is_file()); assert!(toolsdir.join("rpm2kmodkit").is_file()); assert!(toolsdir.join("rpm2migrations").is_file()); - assert!(toolsdir.join("metadata.spec").is_file()); // Check that binaries were copied. assert!(toolsdir.join("bottlerocket-variant").is_file()); @@ -123,7 +124,9 @@ async fn test_install_tools() { assert!(toolsdir.join("tuftool").is_file()); // Check that the mtimes match. - let dockerfile_metadata = fs::metadata(toolsdir.join("Dockerfile")).await.unwrap(); + let dockerfile_metadata = fs::metadata(toolsdir.join("build.Dockerfile")) + .await + .unwrap(); let buildsys_metadata = fs::metadata(toolsdir.join("buildsys")).await.unwrap(); let dockerfile_mtime = FileTime::from_last_modification_time(&dockerfile_metadata); let buildsys_mtime = FileTime::from_last_modification_time(&buildsys_metadata); From 945dfa3364fa9d3fb7e52cfd31b44f30e5a2d31b Mon Sep 17 00:00:00 2001 From: "Patrick J.P. Culp" Date: Tue, 14 May 2024 02:12:35 +0000 Subject: [PATCH 2/2] buildsys: add repack-variant target Add the the initial target for repackaging a Bottlerocket variant. This target will skip the traditional workflow of building packages and using `rpm2img`. Instead the build system will point to a yet to be implemented `img2img` tool. Signed-off-by: Patrick J.P. Culp --- docs/design/README.md | 2 +- tools/buildsys/src/args.rs | 27 +++++ tools/buildsys/src/builder.rs | 109 +++++++++++++++++- tools/buildsys/src/lib.rs | 1 + tools/buildsys/src/main.rs | 20 +++- tools/buildsys/src/manifest.rs | 1 + twoliter/build.rs | 3 + twoliter/embedded/Makefile.toml | 14 +++ twoliter/embedded/img2img | 3 + twoliter/embedded/repack.Dockerfile | 58 ++++++++++ .../embedded/repack.Dockerfile.dockerignore | 4 + twoliter/src/tools.rs | 3 + 12 files changed, 241 insertions(+), 4 deletions(-) create mode 100755 twoliter/embedded/img2img create mode 100644 twoliter/embedded/repack.Dockerfile create mode 100644 twoliter/embedded/repack.Dockerfile.dockerignore diff --git a/docs/design/README.md b/docs/design/README.md index fdd486f62..57031dbfc 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -278,7 +278,7 @@ These will not be available and need to be added to the Bottlerocket SDK: The existing build system relies on certain binaries, scripts and static files, such as - buildsys, pubsys, testsys, etc. -- rpm2img, the Dockerfile, and Makefile.toml +- rpm2img, the Dockerfiles, and Makefile.toml These tools and files will be embedded into the Twoliter binary and installed on-the-fly when Twoliter runs. The binaries will be built and embedded using the unstable Cargo feature [bindeps]. diff --git a/tools/buildsys/src/args.rs b/tools/buildsys/src/args.rs index c1eb63776..06df128a4 100644 --- a/tools/buildsys/src/args.rs +++ b/tools/buildsys/src/args.rs @@ -41,6 +41,7 @@ pub(crate) struct Buildsys { pub(crate) enum Command { BuildPackage(Box), BuildVariant(Box), + RepackVariant(Box), } impl Command { @@ -48,6 +49,7 @@ impl Command { match self { Command::BuildPackage(_) => BuildType::Package, Command::BuildVariant(_) => BuildType::Variant, + Command::RepackVariant(_) => BuildType::Repack, } } } @@ -160,6 +162,25 @@ pub(crate) struct BuildVariantArgs { pub(crate) common: Common, } +/// Repack variant from prebuilt images. +#[derive(Debug, Parser)] +pub(crate) struct RepackVariantArgs { + #[arg(long, env = "BUILDSYS_NAME")] + pub(crate) name: String, + + #[arg(long, env = "BUILDSYS_VARIANT")] + pub(crate) variant: String, + + #[arg(long, env = "BUILDSYS_VERSION_BUILD")] + pub(crate) version_build: String, + + #[arg(long, env = "BUILDSYS_VERSION_IMAGE")] + pub(crate) version_image: String, + + #[command(flatten)] + pub(crate) common: Common, +} + /// Returns the environment variables that need to be watched for a given `[BuildType]`. fn sensitive_env_vars(build_type: BuildFlags) -> impl Iterator { REBUILD_VARS @@ -186,6 +207,7 @@ enum BuildFlags { Package = 0b00000001, Kit = 0b00000010, Variant = 0b00000100, + Repack = 0b0001000, } impl From for BuildFlags { @@ -194,6 +216,7 @@ impl From for BuildFlags { BuildType::Package => BuildFlags::Package, BuildType::Kit => BuildFlags::Kit, BuildType::Variant => BuildFlags::Variant, + BuildType::Repack => BuildFlags::Repack, } } } @@ -206,17 +229,21 @@ impl BuildFlags { } } +#[allow(dead_code)] +const REPACK: u8 = BuildFlags::Repack as u8; const PACKAGE: u8 = BuildFlags::Package as u8; const VARIANT: u8 = BuildFlags::Variant as u8; #[test] fn build_type_includes_test() { // true + assert!(BuildFlags::Repack.includes(REPACK)); assert!(BuildFlags::Package.includes(PACKAGE | VARIANT)); assert!(BuildFlags::Variant.includes(VARIANT)); assert!(BuildFlags::Variant.includes(VARIANT | PACKAGE)); // false + assert!(!BuildFlags::Repack.includes(PACKAGE | VARIANT)); assert!(!BuildFlags::Package.includes(VARIANT)); assert!(!BuildFlags::Variant.includes(PACKAGE)); assert!(!BuildFlags::Variant.includes(32)); diff --git a/tools/buildsys/src/builder.rs b/tools/buildsys/src/builder.rs index 707ea6960..d1b5fc8ed 100644 --- a/tools/buildsys/src/builder.rs +++ b/tools/buildsys/src/builder.rs @@ -6,7 +6,7 @@ the repository's top-level Dockerfile. */ pub(crate) mod error; -use crate::args::{BuildPackageArgs, BuildVariantArgs}; +use crate::args::{BuildPackageArgs, BuildVariantArgs, RepackVariantArgs}; use buildsys::manifest::{ ImageFeature, ImageFormat, ImageLayout, Manifest, PartitionPlan, SupportedArch, }; @@ -86,7 +86,6 @@ lazy_static! { static DOCKER_BUILD_MAX_ATTEMPTS: NonZeroU16 = nonzero!(10u16); -#[allow(dead_code)] enum OutputCleanup { BeforeBuild, None, @@ -224,10 +223,52 @@ impl VariantBuildArgs { } } +struct RepackVariantBuildArgs { + data_image_publish_size_gib: i32, + data_image_size_gib: String, + image_features: HashSet, + image_format: String, + name: String, + os_image_publish_size_gib: String, + os_image_size_gib: String, + partition_plan: String, + variant: String, + version_build: String, + version_image: String, +} + +impl RepackVariantBuildArgs { + fn build_args(&self) -> Vec { + let mut args = Vec::new(); + args.push("--network".into()); + args.push("host".into()); + args.build_arg( + "DATA_IMAGE_PUBLISH_SIZE_GIB", + self.data_image_publish_size_gib.to_string(), + ); + args.build_arg("DATA_IMAGE_SIZE_GIB", &self.data_image_size_gib); + args.build_arg("IMAGE_FORMAT", &self.image_format); + args.build_arg("IMAGE_NAME", &self.name); + args.build_arg("OS_IMAGE_PUBLISH_SIZE_GIB", &self.os_image_publish_size_gib); + args.build_arg("OS_IMAGE_SIZE_GIB", &self.os_image_size_gib); + args.build_arg("PARTITION_PLAN", &self.partition_plan); + args.build_arg("VARIANT", &self.variant); + args.build_arg("BUILD_ID", &self.version_build); + args.build_arg("VERSION_ID", &self.version_image); + + for image_feature in self.image_features.iter() { + args.build_arg(format!("{}", image_feature), "1"); + } + + args + } +} + #[allow(clippy::large_enum_variant)] enum TargetBuildArgs { Package(PackageBuildArgs), Variant(VariantBuildArgs), + Repack(RepackVariantBuildArgs), } impl TargetBuildArgs { @@ -235,6 +276,7 @@ impl TargetBuildArgs { match self { TargetBuildArgs::Package(_) => BuildType::Package, TargetBuildArgs::Variant(_) => BuildType::Variant, + TargetBuildArgs::Repack(_) => BuildType::Repack, } } } @@ -384,6 +426,67 @@ impl DockerBuild { }) } + /// Create a new `DockerBuild` that can repackage a variant image. + pub(crate) fn repack_variant(args: RepackVariantArgs, manifest: &Manifest) -> Result { + let image_layout = manifest.info().image_layout().cloned().unwrap_or_default(); + let ImageLayout { + os_image_size_gib, + data_image_size_gib, + partition_plan, + .. + } = image_layout; + + let (os_image_publish_size_gib, data_image_publish_size_gib) = + image_layout.publish_image_sizes_gib(); + + Ok(Self { + dockerfile: args.common.tools_dir.join("repack.Dockerfile"), + context: args.common.root_dir.clone(), + target: "repack".to_string(), + tag: append_token( + format!( + "buildsys-repack-{variant}-{arch}", + variant = args.variant, + arch = args.common.arch + ), + &args.common.root_dir, + ), + root_dir: args.common.root_dir.clone(), + artifacts_dir: args.common.image_arch_variant_dir, + state_dir: args.common.state_dir, + artifact_name: args.variant.clone(), + common_build_args: CommonBuildArgs::new( + &args.common.root_dir, + args.common.sdk_image, + args.common.arch, + OutputCleanup::None, + ), + target_build_args: TargetBuildArgs::Repack(RepackVariantBuildArgs { + data_image_publish_size_gib, + data_image_size_gib: data_image_size_gib.to_string(), + image_features: manifest.info().image_features().unwrap_or_default(), + image_format: match manifest.info().image_format() { + Some(ImageFormat::Raw) | None => "raw", + Some(ImageFormat::Qcow2) => "qcow2", + Some(ImageFormat::Vmdk) => "vmdk", + } + .to_string(), + name: args.name, + os_image_publish_size_gib: os_image_publish_size_gib.to_string(), + os_image_size_gib: os_image_size_gib.to_string(), + partition_plan: match partition_plan { + PartitionPlan::Split => "split", + PartitionPlan::Unified => "unified", + } + .to_string(), + variant: args.variant, + version_build: args.version_build, + version_image: args.version_image, + }), + secrets_args: secrets_args()?, + }) + } + pub(crate) fn build(&self) -> Result<()> { env::set_current_dir(&self.root_dir).context(error::DirectoryChangeSnafu { path: &self.root_dir, @@ -468,6 +571,7 @@ impl DockerBuild { let mut args = match &self.target_build_args { TargetBuildArgs::Package(p) => p.build_args(), TargetBuildArgs::Variant(v) => v.build_args(), + TargetBuildArgs::Repack(r) => r.build_args(), }; args.build_arg("ARCH", self.common_build_args.arch.to_string()); args.build_arg("GOARCH", self.common_build_args.arch.goarch()); @@ -571,6 +675,7 @@ fn create_marker_dir( BuildType::Package => "packages", BuildType::Kit => "kits", BuildType::Variant => "variants", + BuildType::Repack => "variants", }; let path = [&state_dir.display().to_string(), arch, prefix, name] diff --git a/tools/buildsys/src/lib.rs b/tools/buildsys/src/lib.rs index 36cb5e4d4..d8f10d93c 100644 --- a/tools/buildsys/src/lib.rs +++ b/tools/buildsys/src/lib.rs @@ -6,4 +6,5 @@ pub enum BuildType { Package, Kit, Variant, + Repack, } diff --git a/tools/buildsys/src/main.rs b/tools/buildsys/src/main.rs index 318e0e326..8bf3667bf 100644 --- a/tools/buildsys/src/main.rs +++ b/tools/buildsys/src/main.rs @@ -15,7 +15,7 @@ mod gomod; mod project; mod spec; -use crate::args::{BuildPackageArgs, BuildVariantArgs, Buildsys, Command}; +use crate::args::{BuildPackageArgs, BuildVariantArgs, Buildsys, Command, RepackVariantArgs}; use crate::builder::DockerBuild; use buildsys::manifest::{BundleModule, Manifest, ManifestInfo, SupportedArch}; use cache::LookasideCache; @@ -103,6 +103,7 @@ fn run(args: Buildsys) -> Result<()> { match args.command { Command::BuildPackage(args) => build_package(*args), Command::BuildVariant(args) => build_variant(*args), + Command::RepackVariant(args) => repack_variant(*args), } } @@ -256,6 +257,23 @@ fn build_variant(args: BuildVariantArgs) -> Result<()> { .context(error::BuildAttemptSnafu) } +fn repack_variant(args: RepackVariantArgs) -> Result<()> { + let manifest_file = "Cargo.toml"; + + let manifest = Manifest::new( + args.common.cargo_manifest_dir.join(manifest_file), + &args.common.cargo_metadata_path, + ) + .context(error::ManifestParseSnafu)?; + + supported_arch(manifest.info(), args.common.arch)?; + + DockerBuild::repack_variant(args, &manifest) + .context(error::BuilderInstantiationSnafu)? + .build() + .context(error::BuildAttemptSnafu) +} + /// Ensure that the current arch is supported by the current variant fn supported_arch(manifest: &ManifestInfo, arch: SupportedArch) -> Result<()> { if let Some(supported_arches) = manifest.supported_arches() { diff --git a/tools/buildsys/src/manifest.rs b/tools/buildsys/src/manifest.rs index ffac4dacd..8fbb0299c 100644 --- a/tools/buildsys/src/manifest.rs +++ b/tools/buildsys/src/manifest.rs @@ -504,6 +504,7 @@ fn is_manifest_type(pkg_metadata: &PackageMetadata, manifest_type: BuildType) -> BuildType::Package => metadata_table.get("build-package").is_some(), BuildType::Kit => metadata_table.get("build-kit").is_some(), BuildType::Variant => metadata_table.get("build-variant").is_some(), + BuildType::Repack => unreachable!("Repacking is not defined in manifests"), } } diff --git a/twoliter/build.rs b/twoliter/build.rs index 86d736982..fa940d3f5 100644 --- a/twoliter/build.rs +++ b/twoliter/build.rs @@ -30,8 +30,11 @@ fn main() { paths.copy_file("build.Dockerfile"); paths.copy_file("build.Dockerfile.dockerignore"); paths.copy_file("docker-go"); + paths.copy_file("img2img"); paths.copy_file("imghelper"); paths.copy_file("partyplanner"); + paths.copy_file("repack.Dockerfile"); + paths.copy_file("repack.Dockerfile.dockerignore"); paths.copy_file("rpm2img"); paths.copy_file("rpm2kmodkit"); paths.copy_file("rpm2migrations"); diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index 832014459..84d538dab 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -811,6 +811,20 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${BUILDSYS_OUTPUT_DIR}/latest" ''' ] +[tasks.repack-variant] +dependencies = ["fetch-sdk", "build-sbkeys", "publish-setup"] +script = [ +''' +export PATH="${TWOLITER_TOOLS_DIR}:${PATH}" + +# Parse the variant into its components and set additional variables. +eval "$(bottlerocket-variant)" + +buildsys repack-variant \ + --cargo-manifest-dir variants/${BUILDSYS_VARIANT} +''' +] + [tasks.check-licenses] dependencies = ["fetch"] script = [ diff --git a/twoliter/embedded/img2img b/twoliter/embedded/img2img new file mode 100755 index 000000000..6988d39c9 --- /dev/null +++ b/twoliter/embedded/img2img @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +echo "img2img not yet implemented" >&2 ; exit 1 diff --git a/twoliter/embedded/repack.Dockerfile b/twoliter/embedded/repack.Dockerfile new file mode 100644 index 000000000..46b1f84d5 --- /dev/null +++ b/twoliter/embedded/repack.Dockerfile @@ -0,0 +1,58 @@ +# syntax=docker/dockerfile:1.4.3 +# +# Several commands start with RUN --mount=target=/host, which mounts the docker build +# context (which in practice is the root of the Bottlerocket repository) as a read-only +# filesystem at /host. +ARG SDK + +FROM ${SDK} as imgrepack + +ARG ARCH +ARG VERSION_ID +ARG BUILD_ID +ARG NOCACHE +ARG VARIANT +ARG IMAGE_NAME +ARG IMAGE_FORMAT +ARG OS_IMAGE_SIZE_GIB +ARG DATA_IMAGE_SIZE_GIB +ARG PARTITION_PLAN +ARG OS_IMAGE_PUBLISH_SIZE_GIB +ARG DATA_IMAGE_PUBLISH_SIZE_GIB +ARG UEFI_SECURE_BOOT +ENV VARIANT=${VARIANT} VERSION_ID=${VERSION_ID} BUILD_ID=${BUILD_ID} +WORKDIR /root + +USER root +RUN --mount=target=/host \ + --mount=type=secret,id=PK.crt,target=/root/sbkeys/PK.crt \ + --mount=type=secret,id=KEK.crt,target=/root/sbkeys/KEK.crt \ + --mount=type=secret,id=db.crt,target=/root/sbkeys/db.crt \ + --mount=type=secret,id=vendor.crt,target=/root/sbkeys/vendor.crt \ + --mount=type=secret,id=shim-sign.key,target=/root/sbkeys/shim-sign.key \ + --mount=type=secret,id=shim-sign.crt,target=/root/sbkeys/shim-sign.crt \ + --mount=type=secret,id=code-sign.key,target=/root/sbkeys/code-sign.key \ + --mount=type=secret,id=code-sign.crt,target=/root/sbkeys/code-sign.crt \ + --mount=type=secret,id=config-sign.key,target=/root/sbkeys/config-sign.key \ + --mount=type=secret,id=kms-sign.json,target=/root/.config/aws-kms-pkcs11/config.json \ + --mount=type=secret,id=aws-access-key-id.env,target=/root/.aws/aws-access-key-id.env \ + --mount=type=secret,id=aws-secret-access-key.env,target=/root/.aws/aws-secret-access-key.env \ + --mount=type=secret,id=aws-session-token.env,target=/root/.aws/aws-session-token.env \ + /host/build/tools/img2img \ + --input-dir="/host/build/images/${ARCH}-${VARIANT}/${VERSION_ID}-${BUILD_ID}" \ + --output-dir=/local/output \ + --output-fmt="${IMAGE_FORMAT}" \ + --os-image-size-gib="${OS_IMAGE_SIZE_GIB}" \ + --data-image-size-gib="${DATA_IMAGE_SIZE_GIB}" \ + --os-image-publish-size-gib="${OS_IMAGE_PUBLISH_SIZE_GIB}" \ + --data-image-publish-size-gib="${DATA_IMAGE_PUBLISH_SIZE_GIB}" \ + --partition-plan="${PARTITION_PLAN}" \ + --ovf-template="/host/variants/${VARIANT}/template.ovf" \ + ${UEFI_SECURE_BOOT:+--with-uefi-secure-boot=yes} \ + && echo ${NOCACHE} + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# Copies the repackaged artifacts to their expected location so that buildsys can find them +# and copy them out. +FROM scratch AS repack +COPY --from=imgrepack /local/output/. /output/ diff --git a/twoliter/embedded/repack.Dockerfile.dockerignore b/twoliter/embedded/repack.Dockerfile.dockerignore new file mode 100644 index 000000000..59427247c --- /dev/null +++ b/twoliter/embedded/repack.Dockerfile.dockerignore @@ -0,0 +1,4 @@ +/ +!/build/images/* +!/build/tools/* +!/variants/*/template*.ovf diff --git a/twoliter/src/tools.rs b/twoliter/src/tools.rs index 1cf2ce62f..c80a17c42 100644 --- a/twoliter/src/tools.rs +++ b/twoliter/src/tools.rs @@ -108,9 +108,12 @@ async fn test_install_tools() { assert!(toolsdir.join("build.Dockerfile").is_file()); assert!(toolsdir.join("build.Dockerfile.dockerignore").is_file()); assert!(toolsdir.join("docker-go").is_file()); + assert!(toolsdir.join("img2img").is_file()); assert!(toolsdir.join("imghelper").is_file()); assert!(toolsdir.join("metadata.spec").is_file()); assert!(toolsdir.join("partyplanner").is_file()); + assert!(toolsdir.join("repack.Dockerfile").is_file()); + assert!(toolsdir.join("repack.Dockerfile.dockerignore").is_file()); assert!(toolsdir.join("rpm2img").is_file()); assert!(toolsdir.join("rpm2kmodkit").is_file()); assert!(toolsdir.join("rpm2migrations").is_file());