Expose xtask as a binary entry and streamline "target/architecture" input

Cargo.toml

@@ -81,6 +81,12 @@ serde_dynamo = { version = "4", features = [
   "aws-sdk-dynamodb+0_22",
 ], optional = true }
 uuid = {version = "1", features = ["serde", "v4"], optional = true}
+cargo-options = { version = "0.2.1", optional = true }
+cargo-zigbuild = { version = "0.10.3", optional = true }
+cargo_metadata = { version = "0.15", optional = true }
+xflags = { version = "0.2", optional = true }
+xshell = { version = "0.1", optional = true }
+
 
 [dev-dependencies]
 anyhow = "1"
@@ -98,6 +104,16 @@ serde_json = { version = "1" }
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 
 [features]
+xtask = [
+  "dep:anyhow",
+  "dep:cargo-options",
+  "dep:cargo_metadata",
+  "dep:cargo-zigbuild",
+  "dep:serde_json",
+  "dep:serde",
+  "dep:xflags",
+  "dep:xshell",
+]
 aws_lambda_utils = [
   "dep:tower-service",
   "dep:lambda_runtime",

rust-toolchain

@@ -0,0 +1 @@
+1.65.0

src/build/dist.rs

@@ -0,0 +1,93 @@
+use std::{io::BufRead, path::PathBuf};
+
+use anyhow::{Context, Result};
+use cargo_zigbuild::Build;
+use serde::Deserialize;
+use xshell::{cmd, cp, mkdir_p, pushenv};
+
+use crate::build::flags::{self, Toolchain};
+
+#[derive(Deserialize)]
+struct CargoMessageFormat {
+    executable: Option<String>,
+}
+
+impl flags::Dist {
+    pub(crate) fn run(self) -> Result<()> {
+        let arch = self.architecture;
+        let package = self.package;
+
+        // Without this, xtask would be part of the zigcc wrapper linker script. We need to default to cargo-zigbuild, so that cargo-zigbuild is invoked
+        let _f = pushenv("CARGO_BIN_EXE_cargo-zigbuild", "cargo-zigbuild");
+
+        let mut build_cmd = cargo_zigbuild::Build::build_command(&Build {
+            cargo: cargo_options::Build {
+                common: cargo_options::CommonOptions {
+                    locked: true,
+                    profile: Some("dist".to_string()),
+                    target: vec![arch.target_triple()],
+                    message_format: vec!["json".to_string()],
+                    ..Default::default()
+                },
+                packages: vec![package],
+                ..Default::default()
+            },
+            disable_zig_linker: false,
+        })?;
+
+        println!("Running {:#?}", build_cmd);
+
+        let child = build_cmd.output().context("Failed to run cargo build")?;
+        if !child.status.success() {
+            println!("{}", String::from_utf8_lossy(&child.stdout));
+            eprintln!("{}", String::from_utf8_lossy(&child.stderr));
+            std::process::exit(child.status.code().unwrap_or(1));
+        }
+        let stdout = child.stdout;
+
+        // Find all executables generated by the package.
+        // All executables are outputted to the same target dir.
+        // To find which executables were generated by the package, parse the `executable` field.
+        let mut executables_generated: Vec<PathBuf> = vec![];
+
+        for line in stdout.lines() {
+            if let Ok(CargoMessageFormat {
+                executable: Some(exec),
+            }) = serde_json::from_str::<CargoMessageFormat>(&line?)
+            {
+                executables_generated.push(exec.into());
+            }
+        }
+
+        assert!(
+            !executables_generated.is_empty(),
+            "No executables were generated"
+        );
+
+        // Compress the debug sections to reduce the file size of the binaries.
+        // This means less data to transfer to S3, and to the lambda service.
+        // Panic backtraces with RUST_BACKTRACE=1 still work, as the debug information is still there,
+        // just compressed.
+        let objcopy = arch.objcopy();
+        for exe in &executables_generated {
+            cmd!("{objcopy} {exe} --compress-debug-sections").run()?;
+        }
+
+        let out_dir = self.out_dir;
+
+        // Copy each executable to its own folder in dist named after the generated executable.
+        // The copied executable will be named bootstrap to match AWS Lambda requirements
+        for exe in executables_generated {
+            let executable_dir = format!(
+                "{}/{}",
+                out_dir.display(),
+                exe.file_name().unwrap().to_str().unwrap()
+            );
+            mkdir_p(&executable_dir)?;
+
+            cp(exe, format!("{}/{}", executable_dir, "bootstrap"))?;
+        }
+
+        Ok(())
+    }
+}

src/build/flags.rs

@@ -0,0 +1,112 @@
+use std::path::PathBuf;
+use std::str::FromStr;
+
+pub trait Toolchain {
+    fn target_triple(&self) -> String;
+    fn objcopy(&self) -> String;
+}
+
+#[derive(Debug, Clone)]
+pub struct Graviton2;
+#[derive(Debug, Clone)]
+pub struct X86;
+
+#[derive(Debug, Clone)]
+pub enum Architecture {
+    Graviton2(Graviton2),
+    X86(X86),
+}
+
+impl Default for Architecture {
+    fn default() -> Self {
+        Architecture::X86(X86)
+    }
+}
+
+impl Toolchain for Architecture {
+    fn target_triple(&self) -> String {
+        match self {
+            Architecture::Graviton2(a) => a.target_triple(),
+            Architecture::X86(a) => a.target_triple(),
+        }
+    }
+
+    fn objcopy(&self) -> String {
+        match self {
+            Architecture::Graviton2(a) => a.objcopy(),
+            Architecture::X86(a) => a.objcopy(),
+        }
+    }
+}
+
+impl Toolchain for Graviton2 {
+    fn target_triple(&self) -> String {
+        "aarch64-unknown-linux-gnu.2.26".into()
+    }
+
+    fn objcopy(&self) -> String {
+        "aarch64-linux-gnu-objcopy".into()
+    }
+}
+
+impl Toolchain for X86 {
+    fn target_triple(&self) -> String {
+        "x86_64-unknown-linux-gnu.2.26".into()
+    }
+
+    fn objcopy(&self) -> String {
+        // Well - not entirely true - depends on the machine
+        "objcopy".into()
+    }
+}
+
+impl FromStr for Architecture {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "graviton2" => Ok(Architecture::Graviton2(Graviton2)),
+            "x86" => Ok(Architecture::X86(X86)),
+            _ => Err(format!("Unknown architecture: {}", s)),
+        }
+    }
+}
+
+// TODO: We may want to use clap instead
+xflags::xflags! {
+    cmd xtask
+    {
+        default cmd help {
+            /// Print help information.
+            optional -h, --help
+        }
+
+        cmd dist
+            /// The workspace package to build. e.g ms-auth-rs
+            required package: String
+
+            /// Architecture to build for
+            required architecture: Architecture
+        {
+            /// Directory where generated executables are copied.
+            /// Filepath must be relative to workspace root. E.g. ./ms-auth-rs/target/dist/ms-auth-rs
+            required --out-dir out_dir: PathBuf
+        }
+
+        cmd napi
+        {
+            /// The workspace package to build. e.g. modules_ui_base_version
+            repeated --package package: String
+            /// Path where generated .node file is copied.
+            /// Filepath must be relative to workspace root. E.g. ./ui-auth/target/dist/ui_base_version.node
+            repeated --out-path out_path: PathBuf
+
+            /// Architecture to compile for. Can include zig glibc suffix. Defaults to host target.
+            optional --architecture architecture: Architecture
+
+            /// Compile in release mode
+            optional --release
+        }
+
+    }
+}

src/build/mod.rs

@@ -0,0 +1,3 @@
+pub mod dist;
+pub mod flags;
+pub mod napi;

src/build/napi.rs

@@ -0,0 +1,108 @@
+use anyhow::{Context, Result};
+use cargo_metadata::Message;
+use cargo_zigbuild::Build;
+use xshell::{cmd, cp, mkdir_p, pushenv};
+
+use crate::build::flags::{self, Toolchain};
+
+impl flags::Napi {
+    pub(crate) fn run(self) -> Result<()> {
+        let arch = self.architecture.unwrap_or_default();
+
+        let packages = self.package.clone();
+        let out_paths = self.out_path.clone();
+        let packages_with_out_paths = self
+            .package
+            .into_iter()
+            .zip(self.out_path.into_iter())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            packages.len(),
+            out_paths.len(),
+            "Please provide the same number of out paths as packages"
+        );
+
+        // Without this, xtask would be part of the zigcc wrapper linker script. We need to default to cargo-zigbuild, so that cargo-zigbuild is invoked
+        let _f = pushenv("CARGO_BIN_EXE_cargo-zigbuild", "cargo-zigbuild");
+
+        let is_release_build = self.release;
+        let profile = if is_release_build {
+            Some("dist".to_string())
+        } else {
+            None
+        };
+
+        let mut build_cmd = cargo_zigbuild::Build::build_command(&Build {
+            cargo: cargo_options::Build {
+                common: cargo_options::CommonOptions {
+                    locked: true,
+                    profile,
+                    target: vec![arch.target_triple()],
+                    message_format: vec!["json".to_string()],
+                    ..Default::default()
+                },
+                packages: packages.clone(),
+                ..Default::default()
+            },
+            disable_zig_linker: false,
+        })?;
+
+        println!("Running {:#?}", build_cmd);
+
+        let child = build_cmd.output().context("Failed to run cargo build")?;
+        if !child.status.success() {
+            println!("{}", String::from_utf8_lossy(&child.stdout));
+            eprintln!("{}", String::from_utf8_lossy(&child.stderr));
+            std::process::exit(child.status.code().unwrap_or(1));
+        }
+
+        let messages = Message::parse_stream(child.stdout.as_slice());
+
+        // Find libs generated by the packages.
+        // All libs are outputted to the same target dir.
+        let interesting_messages = messages
+            .into_iter()
+            .map(|x| x.unwrap())
+            .filter_map(|x| match x {
+                Message::CompilerArtifact(a) => Some(a),
+                _ => None,
+            })
+            .filter(|x| packages.iter().any(|p| &x.target.name == p))
+            .collect::<Vec<_>>();
+
+        assert_eq!(
+            interesting_messages.len(),
+            out_paths.len(),
+            "Number of generated libs is not same as expected output paths"
+        );
+
+        for (pkg, out_path) in packages_with_out_paths {
+            let message = interesting_messages
+                .iter()
+                .find(|m| m.target.name == pkg)
+                .unwrap_or_else(|| panic!("Could not find compiler message for {}", pkg));
+
+            let filenames = message.filenames.clone();
+            assert_eq!(
+                filenames.len(),
+                1,
+                "One package should only generate a single file"
+            );
+            let lib_generated = &filenames[0];
+            // Compress the debug sections to reduce the file size of the lib.
+            // This means less data to transfer to S3, and to the lambda service.
+            // Panic backtraces with RUST_BACKTRACE=1 still work, as the debug information is still there,
+            // just compressed.
+            if is_release_build {
+                let objcopy = arch.objcopy();
+                cmd!("{objcopy} {lib_generated} --compress-debug-sections").run()?;
+            }
+
+            let lib_dir = out_path.ancestors().nth(1).unwrap();
+            mkdir_p(lib_dir)?;
+            cp(lib_generated, out_path)?;
+        }
+
+        Ok(())
+    }
+}

src/main.rs

@@ -0,0 +1,32 @@
+mod build;
+
+use std::{
+    env,
+    path::{Path, PathBuf},
+};
+
+use anyhow::Result;
+use xshell::pushd;
+
+fn main() -> Result<()> {
+    let _d = pushd(project_root())?;
+    let flags = build::flags::Xtask::from_env()?;
+    match flags.subcommand {
+        build::flags::XtaskCmd::Help(_) => {
+            println!("{}", build::flags::Xtask::HELP);
+            Ok(())
+        }
+        build::flags::XtaskCmd::Dist(cmd) => cmd.run(),
+        build::flags::XtaskCmd::Napi(cmd) => cmd.run(),
+    }
+}
+
+fn project_root() -> PathBuf {
+    Path::new(
+        &env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| env!("CARGO_MANIFEST_DIR").to_owned()),
+    )
+    .ancestors()
+    .nth(1)
+    .unwrap()
+    .to_path_buf()
+}