feat: compile an entire workspace (shuttle-hq#767)

* feat: compile an entire workspace

* feat: run workspace tests

cargo-shuttle/src/lib.rs

@@ -37,7 +37,7 @@ use git2::{Repository, StatusOptions};
 use ignore::overrides::OverrideBuilder;
 use ignore::WalkBuilder;
 use shuttle_common::models::{project, secret};
-use shuttle_service::builder::{build_crate, Runtime};
+use shuttle_service::builder::{build_workspace, Runtime};
 use std::fmt::Write;
 use strum::IntoEnumIterator;
 use tar::Builder;
@@ -448,7 +448,7 @@ impl Shuttle {
             working_directory.display()
         );
 
-        let runtime = build_crate(working_directory, run_args.release, tx).await?;
+        let runtimes = build_workspace(working_directory, run_args.release, tx).await?;
 
         trace!("loading secrets");
 
@@ -473,7 +473,7 @@ impl Shuttle {
 
         let service_name = self.ctx.project_name().to_string();
 
-        let (is_wasm, executable_path) = match runtime {
+        let (is_wasm, executable_path) = match runtimes[0].clone() {
             Runtime::Next(path) => (true, path),
             Runtime::Alpha(path) => (false, path),
         };

deployer/src/deployment/queue.rs

@@ -12,7 +12,7 @@ use crossbeam_channel::Sender;
 use opentelemetry::global;
 use serde_json::json;
 use shuttle_common::claims::Claim;
-use shuttle_service::builder::{build_crate, get_config, Runtime};
+use shuttle_service::builder::{build_workspace, get_config, Runtime};
 use tokio::time::{sleep, timeout};
 use tracing::{debug, debug_span, error, info, instrument, trace, warn, Instrument, Span};
 use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -27,7 +27,7 @@ use std::time::Duration;
 
 use cargo::core::compiler::{CompileMode, MessageFormat};
 use cargo::core::Workspace;
-use cargo::ops::{CompileOptions, TestOptions};
+use cargo::ops::{self, CompileOptions, TestOptions};
 use flate2::read::GzDecoder;
 use tar::Archive;
 use tokio::fs;
@@ -332,9 +332,11 @@ async fn build_deployment(
     project_path: &Path,
     tx: crossbeam_channel::Sender<Message>,
 ) -> Result<Runtime> {
-    build_crate(project_path, true, tx)
+    let runtimes = build_workspace(project_path, true, tx)
         .await
-        .map_err(|e| Error::Build(e.into()))
+        .map_err(|e| Error::Build(e.into()))?;
+
+    Ok(runtimes[0].clone())
 }
 
 #[instrument(skip(project_path, tx))]
@@ -382,13 +384,15 @@ async fn run_pre_deploy_tests(
     // Build tests with a maximum of 4 workers.
     compile_opts.build_config.jobs = 4;
 
+    compile_opts.spec = ops::Packages::All;
+
     let opts = TestOptions {
         compile_opts,
         no_run: false,
         no_fail_fast: false,
     };
 
-    cargo::ops::run_tests(&ws, &opts, &[]).map_err(TestError::Failed)
+    ops::run_tests(&ws, &opts, &[]).map_err(TestError::Failed)
 }
 
 /// This will store the path to the executable for each runtime, which will be the users project with

runtime/tests/integration/helpers.rs

@@ -14,7 +14,7 @@ use shuttle_proto::{
     },
     runtime::{self, runtime_client::RuntimeClient},
 };
-use shuttle_service::builder::{build_crate, Runtime};
+use shuttle_service::builder::{build_workspace, Runtime};
 use tonic::{
     transport::{Channel, Server},
     Request, Response, Status,
@@ -37,11 +37,11 @@ pub async fn spawn_runtime(project_path: String, service_name: &str) -> Result<T
     let runtime_address = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), runtime_port);
 
     let (tx, _) = crossbeam_channel::unbounded();
-    let runtime = build_crate(Path::new(&project_path), false, tx).await?;
+    let runtimes = build_workspace(Path::new(&project_path), false, tx).await?;
 
     let secrets: HashMap<String, String> = Default::default();
 
-    let (is_wasm, bin_path) = match runtime {
+    let (is_wasm, bin_path) = match runtimes[0].clone() {
         Runtime::Next(path) => (true, path),
         Runtime::Alpha(path) => (false, path),
     };

service/src/builder.rs

@@ -2,8 +2,8 @@ use std::path::{Path, PathBuf};
 
 use anyhow::{anyhow, bail, Context};
 use cargo::core::compiler::{CompileKind, CompileMode, CompileTarget, MessageFormat};
-use cargo::core::{Manifest, Shell, Summary, Verbosity, Workspace};
-use cargo::ops::{clean, compile, CleanOptions, CompileOptions};
+use cargo::core::{Package, Shell, Verbosity, Workspace};
+use cargo::ops::{self, clean, compile, CleanOptions, CompileOptions};
 use cargo::util::homedir;
 use cargo::util::interning::InternedString;
 use cargo::Config;
@@ -12,20 +12,21 @@ use crossbeam_channel::Sender;
 use pipe::PipeWriter;
 use tracing::{error, trace};
 
-use crate::NEXT_NAME;
+use crate::{NEXT_NAME, RUNTIME_NAME};
 
+#[derive(Clone, Debug, Eq, PartialEq)]
 /// How to run/build the project
 pub enum Runtime {
     Next(PathBuf),
     Alpha(PathBuf),
 }
 
 /// Given a project directory path, builds the crate
-pub async fn build_crate(
+pub async fn build_workspace(
     project_path: &Path,
     release_mode: bool,
     tx: Sender<Message>,
-) -> anyhow::Result<Runtime> {
+) -> anyhow::Result<Vec<Runtime>> {
     let (read, write) = pipe::pipe();
     let project_path = project_path.to_owned();
 
@@ -49,29 +50,51 @@ pub async fn build_crate(
 
     let config = get_config(write)?;
     let manifest_path = project_path.join("Cargo.toml");
-    let mut ws = Workspace::new(&manifest_path, &config)?;
+    let ws = Workspace::new(&manifest_path, &config)?;
+    check_no_panic(&ws)?;
 
-    let current = ws.current_mut().map_err(|_| anyhow!("A Shuttle project cannot have a virtual manifest file - please ensure the `package` table is present in your Cargo.toml file."))?;
+    let mut alpha_packages = Vec::new();
+    let mut next_packages = Vec::new();
 
-    let summary = current.manifest_mut().summary_mut();
-    let is_next = is_next(summary);
+    for member in ws.members() {
+        if is_next(member) {
+            ensure_cdylib(member)?;
+            next_packages.push(member.name().to_string());
+        } else if is_alpha(member) {
+            ensure_binary(member)?;
+            alpha_packages.push(member.name().to_string());
+        }
+    }
 
-    if !is_next {
-        ensure_binary(current.manifest())?;
-    } else {
-        ensure_cdylib(current.manifest_mut())?;
+    let mut runtimes = Vec::new();
+
+    if !alpha_packages.is_empty() {
+        let opts = get_compile_options(&config, alpha_packages, release_mode, false)?;
+        let compilation = compile(&ws, &opts)?;
+
+        let mut alpha_binaries = compilation
+            .binaries
+            .iter()
+            .map(|binary| Runtime::Alpha(binary.path.clone()))
+            .collect();
+
+        runtimes.append(&mut alpha_binaries);
     }
 
-    check_no_panic(&ws)?;
+    if !next_packages.is_empty() {
+        let opts = get_compile_options(&config, next_packages, release_mode, true)?;
+        let compilation = compile(&ws, &opts)?;
 
-    let opts = get_compile_options(&config, release_mode, is_next)?;
-    let compilation = compile(&ws, &opts)?;
+        let mut next_libraries = compilation
+            .cdylibs
+            .iter()
+            .map(|binary| Runtime::Next(binary.path.clone()))
+            .collect();
 
-    Ok(if is_next {
-        Runtime::Next(compilation.cdylibs[0].path.clone())
-    } else {
-        Runtime::Alpha(compilation.binaries[0].path.clone())
-    })
+        runtimes.append(&mut next_libraries);
+    }
+
+    Ok(runtimes)
 }
 
 pub fn clean_crate(project_path: &Path, release_mode: bool) -> anyhow::Result<Vec<String>> {
@@ -138,6 +161,7 @@ pub fn get_config(writer: PipeWriter) -> anyhow::Result<Config> {
 /// Get options to compile in build mode
 fn get_compile_options(
     config: &Config,
+    packages: Vec<String>,
     release_mode: bool,
     wasm: bool,
 ) -> anyhow::Result<CompileOptions> {
@@ -166,44 +190,40 @@ fn get_compile_options(
         CompileKind::Host
     }];
 
+    opts.spec = ops::Packages::Packages(packages);
+
     Ok(opts)
 }
 
-fn is_next(summary: &Summary) -> bool {
-    summary
+fn is_next(package: &Package) -> bool {
+    package
         .dependencies()
         .iter()
         .any(|dependency| dependency.package_name() == NEXT_NAME)
 }
 
+fn is_alpha(package: &Package) -> bool {
+    package
+        .dependencies()
+        .iter()
+        .any(|dependency| dependency.package_name() == RUNTIME_NAME)
+}
+
 /// Make sure the project is a binary for alpha projects.
-fn ensure_binary(manifest: &Manifest) -> anyhow::Result<()> {
-    if manifest.targets().iter().any(|target| target.is_bin()) {
+fn ensure_binary(package: &Package) -> anyhow::Result<()> {
+    if package.targets().iter().any(|target| target.is_bin()) {
         Ok(())
     } else {
         bail!("Your Shuttle project must be a binary.")
     }
 }
 
 /// Make sure "cdylib" is set for shuttle-next projects, else set it if possible.
-fn ensure_cdylib(manifest: &mut Manifest) -> anyhow::Result<()> {
-    if let Some(target) = manifest
-        .targets_mut()
-        .iter_mut()
-        .find(|target| target.is_lib())
-    {
-        if !target.is_cdylib() {
-            *target = cargo::core::manifest::Target::lib_target(
-                target.name(),
-                vec![cargo::core::compiler::CrateType::Cdylib],
-                target.src_path().path().unwrap().to_path_buf(),
-                target.edition(),
-            );
-        }
-
+fn ensure_cdylib(package: &Package) -> anyhow::Result<()> {
+    if package.targets().iter().any(|target| target.is_lib()) {
         Ok(())
     } else {
-        bail!("Your Shuttle project must be a library. Please add `[lib]` to your Cargo.toml file.")
+        bail!("Your Shuttle next project must be a library. Please add `[lib]` to your Cargo.toml file.")
     }
 }
 

service/src/lib.rs

@@ -190,3 +190,4 @@ pub trait Service: Send {
 }
 
 pub const NEXT_NAME: &str = "shuttle-next";
+pub const RUNTIME_NAME: &str = "shuttle-runtime";

service/tests/integration/build_crate.rs

@@ -1,13 +1,13 @@
 use std::path::{Path, PathBuf};
 
-use shuttle_service::builder::{build_crate, Runtime};
+use shuttle_service::builder::{build_workspace, Runtime};
 
 #[tokio::test]
 #[should_panic(expected = "1 job failed")]
 async fn not_shuttle() {
     let (tx, _) = crossbeam_channel::unbounded();
     let project_path = format!("{}/tests/resources/not-shuttle", env!("CARGO_MANIFEST_DIR"));
-    build_crate(Path::new(&project_path), false, tx)
+    build_workspace(Path::new(&project_path), false, tx)
         .await
         .unwrap();
 }
@@ -17,7 +17,7 @@ async fn not_shuttle() {
 async fn not_bin() {
     let (tx, _) = crossbeam_channel::unbounded();
     let project_path = format!("{}/tests/resources/not-bin", env!("CARGO_MANIFEST_DIR"));
-    match build_crate(Path::new(&project_path), false, tx).await {
+    match build_workspace(Path::new(&project_path), false, tx).await {
         Ok(_) => {}
         Err(e) => panic!("{}", e.to_string()),
     }
@@ -28,13 +28,14 @@ async fn is_bin() {
     let (tx, _) = crossbeam_channel::unbounded();
     let project_path = format!("{}/tests/resources/is-bin", env!("CARGO_MANIFEST_DIR"));
 
-    assert!(matches!(
-        build_crate(Path::new(&project_path), false, tx).await,
-        Ok(Runtime::Alpha(_))
-    ));
-    assert!(PathBuf::from(project_path)
-        .join("target/debug/is-bin")
-        .exists());
+    assert_eq!(
+        build_workspace(Path::new(&project_path), false, tx)
+            .await
+            .unwrap(),
+        vec![Runtime::Alpha(
+            PathBuf::from(project_path).join("target/debug/is-bin")
+        )]
+    );
 }
 
 #[tokio::test]
@@ -45,7 +46,24 @@ async fn not_found() {
         "{}/tests/resources/non-existing",
         env!("CARGO_MANIFEST_DIR")
     );
-    build_crate(Path::new(&project_path), false, tx)
+    build_workspace(Path::new(&project_path), false, tx)
         .await
         .unwrap();
 }
+
+// Test that alpha and next projects are compiled correctly. Any shared library crates should not be compiled too
+#[tokio::test]
+async fn workspace() {
+    let (tx, _) = crossbeam_channel::unbounded();
+    let project_path = format!("{}/tests/resources/workspace", env!("CARGO_MANIFEST_DIR"));
+
+    assert_eq!(
+        build_workspace(Path::new(&project_path), false, tx)
+            .await
+            .unwrap(),
+        vec![
+            Runtime::Alpha(PathBuf::from(&project_path).join("target/debug/alpha")),
+            Runtime::Next(PathBuf::from(&project_path).join("target/wasm32-wasi/debug/next.wasm"))
+        ]
+    );
+}

service/tests/resources/workspace/Cargo.toml

@@ -0,0 +1,7 @@
+[workspace]
+
+members = [
+    "alpha",
+    "next",
+    "shared",
+]

service/tests/resources/workspace/alpha/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "alpha"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+axum = "0.6.0"
+shared = { path = "../shared", version = "0.1.0" }
+shuttle-axum = { path = "../../../../../services/shuttle-axum" }
+shuttle-runtime = { path = "../../../../../runtime" }
+tokio = { version = "1.22.0" }

service/tests/resources/workspace/alpha/src/main.rs

@@ -0,0 +1,12 @@
+use axum::{routing::get, Router};
+
+async fn hello_world() -> &'static str {
+    shared::hello()
+}
+
+#[shuttle_runtime::main]
+async fn axum() -> shuttle_axum::ShuttleAxum {
+    let router = Router::new().route("/hello", get(hello_world));
+
+    Ok(router.into())
+}

service/tests/resources/workspace/next/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "next"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = [ "cdylib" ]
+
+[dependencies]
+shared = { path = "../shared", version = "0.1.0" }
+shuttle-next = { path = "../../../../../services/shuttle-next" }
+tracing = "0.1.37"
+futures = "0.3.25"