Save

Author: andygrove

.dockerignore

@@ -21,4 +21,6 @@
 # This setup requires to all of our docker containers have arrow's source
 # as a mounted directory.
 
+ci
+dev
 **/target/*

dev/docker/rust-base.dockerfile

@@ -23,7 +23,7 @@
 
 
 # Base image extends debian:buster-slim
-FROM rust:1.49.0-buster AS builder
+FROM rust:1.51.0-buster AS builder
 
 RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl
 

dev/docker/rust.dockerfile

@@ -22,41 +22,57 @@
 # as a mounted directory.
 
 ARG RELEASE_FLAG=--release
-FROM ballistacompute/rust-base:0.4.0-20210213 AS base
+FROM ballistacompute/rust-base:0.4.2-SNAPSHOT AS base
 WORKDIR /tmp/ballista
 RUN apt-get -y install cmake
 RUN cargo install cargo-chef 
 
 FROM base as planner
-ADD datafusion .
-ADD ballista .
+RUN mkdir /tmp/ballista/ballista
+RUN mkdir /tmp/ballista/benchmarks
+RUN mkdir /tmp/ballista/datafusion
+RUN mkdir /tmp/ballista/datafusion-examples
+ADD Cargo.toml .
+COPY benchmarks ./benchmarks/
+COPY datafusion ./datafusion/
+COPY datafusion-examples ./datafusion-examples/
+COPY ballista ./ballista/
 RUN cargo chef prepare --recipe-path recipe.json
 
 FROM base as cacher
 COPY --from=planner /tmp/ballista/recipe.json recipe.json
 RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json
 
 FROM base as builder
-ADD datafusion .
-ADD ballista .
+RUN mkdir /tmp/ballista/ballista
+RUN mkdir /tmp/ballista/benchmarks
+RUN mkdir /tmp/ballista/datafusion
+RUN mkdir /tmp/ballista/datafusion-examples
+ADD Cargo.toml .
+COPY benchmarks ./benchmarks/
+COPY datafusion ./datafusion/
+COPY ballista ./ballista/
+COPY datafusion-examples ./datafusion-examples/
 COPY --from=cacher /tmp/ballista/target target
 ARG RELEASE_FLAG=--release
 
 # force build.rs to run to generate configure_me code.
 ENV FORCE_REBUILD='true'
 RUN cargo build $RELEASE_FLAG
+RUN cd ballista/rust && \
+    cargo build $RELEASE_FLAG
 
 # put the executor on /executor (need to be copied from different places depending on FLAG)
 ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-executor /executor; else mv /tmp/ballista/ballista/rust/target/release/ballista-executor /executor; fi
 
 # put the scheduler on /scheduler (need to be copied from different places depending on FLAG)
 ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/ballista/rust/target/release/ballista-scheduler /scheduler; fi
 
 # put the tpch on /tpch (need to be copied from different places depending on FLAG)
 ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/tpch /tpch; else mv /tmp/ballista/ballista/rust/target/release/tpch /tpch; fi
 
 # Copy the binary into a new container for a smaller docker image
 FROM ballistacompute/rust-base:0.4.0-20210213

dev/integration-tests.sh

@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 set -e
+./dev/build-rust-base.sh
 ./dev/build-rust.sh
 pushd ballista/rust/benchmarks/tpch
 ./tpch-gen.sh