Merge #678

678: allow specifying a dockerfile to use for a target and implement "pre-build"ing r=Alexhuszagh a=Emilgardis

Add optional `target.{target}.dockerfile[.file]`, `target.{target}.dockerfile.context` and `target.{target}.dockerfile.build-args` to invoke `docker/podman build` before using an image.

also adds `target.{target}.pre-build` to allow pre-building as per #565

In total, this PR allows a user to do the following:

```toml
[build]
pre-build = [
    "dpkg --add-architecture $CROSS_DEB_ARCH", 
    "apt-get update && apt-get --assume-yes install libssl-dev:$CROSS_DEB_ARCH"
    ]
[target.x86_64-unknown-linux-gnu]
dockerfile = { file = "./Dockerfile.x86_64_ssl", context = "..", build-args = { ARG1 = "foo" }}
image = "my-image:latest"
pre-build = ["echo 'Hello world!'"]
```

fixes #565

Co-authored-by: Emil Gardström <emil.gardstrom@gmail.com>

CHANGELOG.md

@@ -10,7 +10,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - #795 - added images for additional toolchains maintained by cross-rs.
 - #792 - added `CROSS_CONTAINER_IN_CONTAINER` environment variable to replace `CROSS_DOCKER_IN_DOCKER`.
 - #782 - added `build-std` config option, which builds the rust standard library from source if enabled.
-- #775 - forward Cargo exit code to host
+- #678 - Add optional `target.{target}.dockerfile[.file]`, `target.{target}.dockerfile.context` and `target.{target}.dockerfile.build-args` to invoke docker/podman build before using an image.
+- #678 - Add `target.{target}.pre-build` config for running commands before building the image.
 - #772 - added `CROSS_CONTAINER_OPTS` environment variable to replace `DOCKER_OPTS`.
 - #767, #788 - added the `cross-util` and `xtask` commands.
 - #745 - added `thumbv7neon-*` targets.
@@ -27,6 +28,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
+- #775 - forward Cargo exit code to host
 - #762 - re-enabled `x86_64-unknown-dragonfly` target.
 - #747 - reduced android image sizes.
 - #746 - limit image permissions for android images.

Cargo.toml

@@ -31,6 +31,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_ignored = "0.1.2"
 shell-words = "1.1.0"
+sha1_smol = "1.0.0"
 
 [target.'cfg(not(windows))'.dependencies]
 nix = { version = "0.24", default-features = false, features = ["user"] }

README.md

@@ -112,10 +112,22 @@ the default one. Normal Docker behavior applies, so:
 
 - If only `tag` is omitted, then Docker will use the `latest` tag.
 
+#### Dockerfiles
+
+If you're using a custom Dockerfile, you can use `target.{{TARGET}}.dockerfile` to automatically build it
+
+``` toml
+[target.aarch64-unknown-linux-gnu.dockerfile]
+dockerfile = "./path/to/where/the/Dockerfile/resides"
+```
+
+`cross` will build and use the image that was built instead of the default image.
+
 It's recommended to base your custom image on the default Docker image that
 cross uses: `ghcr.io/cross-rs/{{TARGET}}:{{VERSION}}` (where `{{VERSION}}` is cross's version).
 This way you won't have to figure out how to install a cross C toolchain in your
-custom image. Example below:
+custom image.
+
 
 ``` Dockerfile
 FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:latest
@@ -125,8 +137,23 @@ RUN dpkg --add-architecture arm64 && \
     apt-get install --assume-yes libfoo:arm64
 ```
 
+If you want cross to provide the `FROM` instruction, you can do the following
+
+``` Dockerfile
+ARG CROSS_BASE_IMAGE
+FROM $CROSS_BASE_IMAGE
+
+RUN ...
 ```
-$ docker build -t my/image:tag path/to/where/the/Dockerfile/resides
+
+#### Pre-build hook
+
+`cross` enables you to add dependencies and run other necessary commands in the image before using it.
+This action will be added to the used image, so it won't be ran/built every time you use `cross`.
+
+``` toml
+[target.x86_64-unknown-linux-gnu]
+pre-build = ["dpkg --add-architecture arm64 && apt-get update && apt-get install --assume-yes libfoo:arm64"]
 ```
 
 ### Docker in Docker

docs/cross_toml.md

@@ -1,6 +1,7 @@
 The `cross` configuration in the `Cross.toml` file, can contain the following elements:
 
 # `build`
+
 The `build` key allows you to set global variables, e.g.:
 
 ```toml
@@ -11,6 +12,7 @@ default-target = "x86_64-unknown-linux-gnu"
 ```
 
 # `build.env`
+
 With the `build.env` key you can globally set volumes that should be mounted
 in the Docker container or environment variables that should be passed through.
 For example:
@@ -22,17 +24,20 @@ passthrough = ["IMPORTANT_ENV_VARIABLES"]
 ```
 
 # `target.TARGET`
+
 The `target` key allows you to specify parameters for specific compilation targets.
 
 ```toml
 [target.aarch64-unknown-linux-gnu]
 xargo = false
 build-std = false
 image = "test-image"
+pre-build = ["apt-get update"]
 runner = "custom-runner"
 ```
 
 # `target.TARGET.env`
+
 The `target` key allows you to specify environment variables that should be used for a specific compilation target.
 This is similar to `build.env`, but allows you to be more specific per target.
 
@@ -41,3 +46,19 @@ This is similar to `build.env`, but allows you to be more specific per target.
 volumes = ["VOL1_ARG", "VOL2_ARG"]
 passthrough = ["IMPORTANT_ENV_VARIABLES"]
 ```
+
+# `target.TARGET.dockerfile`
+
+```toml
+[target.x86_64-unknown-linux-gnu.dockerfile]
+file = "./Dockerfile" # The dockerfile to use relative to the `Cargo.toml`
+context = "." # What folder to run the build script in
+build-args = { ARG1 = "foo" } # https://docs.docker.com/engine/reference/builder/#arg
+```
+
+also supports
+
+```toml
+[target.x86_64-unknown-linux-gnu]
+dockerfile = "./Dockerfile"
+```

src/bin/commands/images.rs

@@ -186,7 +186,7 @@ fn remove_images(
     }
     command.args(images);
     if execute {
-        command.run(verbose).map_err(Into::into)
+        command.run(verbose, false).map_err(Into::into)
     } else {
         println!("{:?}", command);
         Ok(())

src/cargo.rs

@@ -148,7 +148,9 @@ pub fn cargo_metadata_with_args(
 
 /// Pass-through mode
 pub fn run(args: &[String], verbose: bool) -> Result<ExitStatus, CommandError> {
-    Command::new("cargo").args(args).run_and_get_status(verbose)
+    Command::new("cargo")
+        .args(args)
+        .run_and_get_status(verbose, false)
 }
 
 /// run cargo and get the output, does not check the exit status

src/config.rs

@@ -68,6 +68,23 @@ impl Environment {
         self.get_target_var(target, "IMAGE")
     }
 
+    fn dockerfile(&self, target: &Target) -> Option<String> {
+        let res = self.get_values_for("DOCKERFILE", target, |s| s.to_string());
+        res.0.or(res.1)
+    }
+
+    fn dockerfile_context(&self, target: &Target) -> Option<String> {
+        let res = self.get_values_for("DOCKERFILE_CONTEXT", target, |s| s.to_string());
+        res.0.or(res.1)
+    }
+
+    fn pre_build(&self, target: &Target) -> Option<Vec<String>> {
+        let res = self.get_values_for("PRE_BUILD", target, |v| {
+            v.split('\n').map(String::from).collect()
+        });
+        res.0.or(res.1)
+    }
+
     fn runner(&self, target: &Target) -> Option<String> {
         self.get_target_var(target, "RUNNER")
     }
@@ -178,16 +195,34 @@ impl Config {
     fn vec_from_config(
         &self,
         target: &Target,
-        env: impl Fn(&Environment, &Target) -> (Option<Vec<String>>, Option<Vec<String>>),
-        config_build: impl for<'a> Fn(&'a CrossToml) -> Option<&'a [String]>,
-        config_target: impl for<'a> Fn(&'a CrossToml, &Target) -> Option<&'a [String]>,
-    ) -> Result<Vec<String>> {
+        env: impl for<'a> Fn(&'a Environment, &Target) -> (Option<Vec<String>>, Option<Vec<String>>),
+        config: impl for<'a> Fn(&'a CrossToml, &Target) -> (Option<&'a [String]>, Option<&'a [String]>),
+        sum: bool,
+    ) -> Result<Option<Vec<String>>> {
         let (env_build, env_target) = env(&self.env, target);
 
-        let mut collected = self.sum_of_env_toml_values(env_build, config_build)?;
-        collected.extend(self.sum_of_env_toml_values(env_target, |t| config_target(t, target))?);
+        if sum {
+            return self.sum_of_env_toml_values(env_target, |t| config(t, target));
+        } else if let Some(env_target) = env_target {
+            return Ok(Some(env_target));
+        }
+
+        let (build, target) = self
+            .toml
+            .as_ref()
+            .map(|t| config(t, target))
+            .unwrap_or_default();
+
+        // FIXME: let expression
+        if target.is_none() && env_build.is_some() {
+            return Ok(env_build);
+        }
 
-        Ok(collected)
+        if target.is_none() {
+            Ok(build.map(ToOwned::to_owned))
+        } else {
+            Ok(target.map(ToOwned::to_owned))
+        }
     }
 
     #[cfg(test)]
@@ -211,22 +246,17 @@ impl Config {
         self.string_from_config(target, Environment::runner, CrossToml::runner)
     }
 
-    pub fn env_passthrough(&self, target: &Target) -> Result<Vec<String>> {
+    pub fn env_passthrough(&self, target: &Target) -> Result<Option<Vec<String>>> {
         self.vec_from_config(
             target,
             Environment::passthrough,
-            CrossToml::env_passthrough_build,
-            CrossToml::env_passthrough_target,
+            CrossToml::env_passthrough,
+            true,
         )
     }
 
-    pub fn env_volumes(&self, target: &Target) -> Result<Vec<String>> {
-        self.vec_from_config(
-            target,
-            Environment::volumes,
-            CrossToml::env_volumes_build,
-            CrossToml::env_volumes_target,
-        )
+    pub fn env_volumes(&self, target: &Target) -> Result<Option<Vec<String>>> {
+        self.vec_from_config(target, Environment::volumes, CrossToml::env_volumes, false)
     }
 
     pub fn target(&self, target_list: &TargetList) -> Option<Target> {
@@ -238,19 +268,78 @@ impl Config {
             .and_then(|t| t.default_target(target_list))
     }
 
+    pub fn dockerfile(&self, target: &Target) -> Result<Option<String>> {
+        self.string_from_config(target, Environment::dockerfile, CrossToml::dockerfile)
+    }
+
+    pub fn dockerfile_context(&self, target: &Target) -> Result<Option<String>> {
+        self.string_from_config(
+            target,
+            Environment::dockerfile_context,
+            CrossToml::dockerfile_context,
+        )
+    }
+
+    pub fn dockerfile_build_args(
+        &self,
+        target: &Target,
+    ) -> Result<Option<HashMap<String, String>>> {
+        // This value does not support env variables
+        self.toml
+            .as_ref()
+            .map_or(Ok(None), |t| Ok(t.dockerfile_build_args(target)))
+    }
+
+    pub fn pre_build(&self, target: &Target) -> Result<Option<Vec<String>>> {
+        self.vec_from_config(
+            target,
+            |e, t| (None, e.pre_build(t)),
+            CrossToml::pre_build,
+            false,
+        )
+    }
+
     fn sum_of_env_toml_values<'a>(
         &'a self,
-        env_values: Option<Vec<String>>,
-        toml_getter: impl FnOnce(&'a CrossToml) -> Option<&'a [String]>,
-    ) -> Result<Vec<String>> {
+        env_values: Option<impl AsRef<[String]>>,
+        toml_getter: impl FnOnce(&'a CrossToml) -> (Option<&'a [String]>, Option<&'a [String]>),
+    ) -> Result<Option<Vec<String>>> {
+        let mut defined = false;
         let mut collect = vec![];
-        if let Some(mut vars) = env_values {
-            collect.append(&mut vars);
-        } else if let Some(toml_values) = self.toml.as_ref().and_then(toml_getter) {
-            collect.extend(toml_values.iter().cloned());
+        if let Some(vars) = env_values {
+            collect.extend(vars.as_ref().iter().cloned());
+            defined = true;
+        } else if let Some((build, target)) = self.toml.as_ref().map(toml_getter) {
+            if let Some(build) = build {
+                collect.extend(build.iter().cloned());
+                defined = true;
+            }
+            if let Some(target) = target {
+                collect.extend(target.iter().cloned());
+                defined = true;
+            }
         }
+        if !defined {
+            Ok(None)
+        } else {
+            Ok(Some(collect))
+        }
+    }
+}
 
-        Ok(collect)
+pub fn opt_merge<I, T: Extend<I> + IntoIterator<Item = I>>(
+    opt1: Option<T>,
+    opt2: Option<T>,
+) -> Option<T> {
+    match (opt1, opt2) {
+        (None, None) => None,
+        (None, Some(opt2)) => Some(opt2),
+        (Some(opt1), None) => Some(opt1),
+        (Some(opt1), Some(opt2)) => {
+            let mut res = opt2;
+            res.extend(opt1);
+            Some(res)
+        }
     }
 }
 
@@ -389,7 +478,8 @@ mod tests {
             let config = Config::new_with(Some(toml(TOML_BUILD_VOLUMES)?), env);
             let expected = vec!["VOLUME1".to_string(), "VOLUME2".into()];
 
-            let result = config.env_volumes(&target()).unwrap();
+            let result = config.env_volumes(&target()).unwrap().unwrap_or_default();
+            dbg!(&result);
             assert!(result.len() == 2);
             assert!(result.contains(&expected[0]));
             assert!(result.contains(&expected[1]));
@@ -404,7 +494,8 @@ mod tests {
             let config = Config::new_with(Some(toml(TOML_BUILD_VOLUMES)?), env);
             let expected = vec!["VOLUME3".to_string(), "VOLUME4".into()];
 
-            let result = config.env_volumes(&target()).unwrap();
+            let result = config.env_volumes(&target()).unwrap().unwrap_or_default();
+            dbg!(&result);
             assert!(result.len() == 2);
             assert!(result.contains(&expected[0]));
             assert!(result.contains(&expected[1]));