From 62fb25e383ebc781a04ef22d50e9148b55220605 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 18 Apr 2021 12:58:07 -0600 Subject: [PATCH 1/2] WIP: Fixing Ballista IT --- .dockerignore | 40 +------------------ ballista/.dockerignore | 18 --------- ballista/docker/README.md | 29 -------------- {ballista/dev => dev}/build-rust-base.sh | 2 +- {ballista/dev => dev}/build-rust.sh | 2 +- {ballista => dev}/docker/rust-base.dockerfile | 0 {ballista => dev}/docker/rust.dockerfile | 6 ++- {ballista/dev => dev}/integration-tests.sh | 2 +- 8 files changed, 8 insertions(+), 91 deletions(-) delete mode 100644 ballista/.dockerignore delete mode 100644 ballista/docker/README.md rename {ballista/dev => dev}/build-rust-base.sh (96%) rename {ballista/dev => dev}/build-rust.sh (96%) rename {ballista => dev}/docker/rust-base.dockerfile (100%) rename {ballista => dev}/docker/rust.dockerfile (97%) rename {ballista/dev => dev}/integration-tests.sh (96%) diff --git a/.dockerignore b/.dockerignore index eb71138c679a..d38b04ecb0e1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,42 +21,4 @@ # This setup requires to all of our docker containers have arrow's source # as a mounted directory. -# exclude everything -** - -# include explicitly -!ci/** -!c_glib/Gemfile -!dev/archery/requirements*.txt -!python/requirements*.txt -!python/manylinux1/** -!python/manylinux2010/** -!r/DESCRIPTION -!ruby/Gemfile -!ruby/red-arrow/Gemfile -!ruby/red-arrow/lib/arrow/version.rb -!ruby/red-arrow/red-arrow.gemspec -!ruby/red-arrow-cuda/Gemfile -!ruby/red-arrow-cuda/lib/arrow-cuda/version.rb -!ruby/red-arrow-cuda/red-arrow-cuda.gemspec -!ruby/red-gandiva/Gemfile -!ruby/red-gandiva/lib/gandiva/version.rb -!ruby/red-gandiva/red-gandiva.gemspec -!ruby/red-parquet/Gemfile -!ruby/red-parquet/lib/parquet/version.rb -!ruby/red-parquet/red-parquet.gemspec -!ruby/red-plasma/Gemfile -!ruby/red-plasma/lib/plasma/version.rb -!ruby/red-plasma/red-plasma.gemspec -!rust/Cargo.toml -!rust/benchmarks/Cargo.toml -!rust/arrow/Cargo.toml -!rust/arrow/benches -!rust/arrow-flight/Cargo.toml -!rust/parquet/Cargo.toml -!rust/parquet/build.rs -!rust/parquet_derive/Cargo.toml -!rust/parquet_derive_test/Cargo.toml -!rust/datafusion/Cargo.toml -!rust/datafusion/benches -!rust/integration-testing/Cargo.toml +**/target/* diff --git a/ballista/.dockerignore b/ballista/.dockerignore deleted file mode 100644 index 3cde49e0a0c4..000000000000 --- a/ballista/.dockerignore +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -rust/**/target diff --git a/ballista/docker/README.md b/ballista/docker/README.md deleted file mode 100644 index 8417d04c4922..000000000000 --- a/ballista/docker/README.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# Ballista Docker Images - -Pre-built docker images are available from [Docker Hub](https://hub.docker.com/orgs/ballistacompute/repositories) but here are the commands to build the images from source. - -Run these commands from the root directory of the project. - -```bash -./dev/build-all.sh -``` - diff --git a/ballista/dev/build-rust-base.sh b/dev/build-rust-base.sh similarity index 96% rename from ballista/dev/build-rust-base.sh rename to dev/build-rust-base.sh index ee4b32c8e690..e424909fb6f1 100755 --- a/ballista/dev/build-rust-base.sh +++ b/dev/build-rust-base.sh @@ -18,4 +18,4 @@ # under the License. BALLISTA_VERSION=0.4.2-SNAPSHOT set -e -docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f docker/rust-base.dockerfile . +docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f dev/docker/rust-base.dockerfile . diff --git a/ballista/dev/build-rust.sh b/dev/build-rust.sh similarity index 96% rename from ballista/dev/build-rust.sh rename to dev/build-rust.sh index 1916f8efbefb..d31c5241c6f1 100755 --- a/ballista/dev/build-rust.sh +++ b/dev/build-rust.sh @@ -21,4 +21,4 @@ BALLISTA_VERSION=0.4.2-SNAPSHOT set -e -docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f docker/rust.dockerfile . +docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f dev/docker/rust.dockerfile . diff --git a/ballista/docker/rust-base.dockerfile b/dev/docker/rust-base.dockerfile similarity index 100% rename from ballista/docker/rust-base.dockerfile rename to dev/docker/rust-base.dockerfile diff --git a/ballista/docker/rust.dockerfile b/dev/docker/rust.dockerfile similarity index 97% rename from ballista/docker/rust.dockerfile rename to dev/docker/rust.dockerfile index 8b06af3dc78f..b0e00908c9f5 100644 --- a/ballista/docker/rust.dockerfile +++ b/dev/docker/rust.dockerfile @@ -28,7 +28,8 @@ RUN apt-get -y install cmake RUN cargo install cargo-chef FROM base as planner -COPY rust . +ADD datafusion . +ADD ballista . RUN cargo chef prepare --recipe-path recipe.json FROM base as cacher @@ -36,7 +37,8 @@ COPY --from=planner /tmp/ballista/recipe.json recipe.json RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json FROM base as builder -COPY rust . +ADD datafusion . +ADD ballista . COPY --from=cacher /tmp/ballista/target target ARG RELEASE_FLAG=--release diff --git a/ballista/dev/integration-tests.sh b/dev/integration-tests.sh similarity index 96% rename from ballista/dev/integration-tests.sh rename to dev/integration-tests.sh index cc34a5ce91f5..9ea6b314fe8b 100755 --- a/ballista/dev/integration-tests.sh +++ b/dev/integration-tests.sh @@ -18,7 +18,7 @@ # under the License. set -e ./dev/build-rust.sh -pushd rust/benchmarks/tpch +pushd ballista/rust/benchmarks/tpch ./tpch-gen.sh docker-compose up -d From a5fcea866c11fa04dc40de1b16d881784b82f2e4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 18 Apr 2021 13:42:21 -0600 Subject: [PATCH 2/2] Save --- .dockerignore | 2 ++ dev/docker/rust-base.dockerfile | 2 +- dev/docker/rust.dockerfile | 32 ++++++++++++++++++++++++-------- dev/integration-tests.sh | 1 + 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index d38b04ecb0e1..9a64a123f735 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,4 +21,6 @@ # This setup requires to all of our docker containers have arrow's source # as a mounted directory. +ci +dev **/target/* diff --git a/dev/docker/rust-base.dockerfile b/dev/docker/rust-base.dockerfile index 4519225d2197..31620b38cf39 100644 --- a/dev/docker/rust-base.dockerfile +++ b/dev/docker/rust-base.dockerfile @@ -23,7 +23,7 @@ # Base image extends debian:buster-slim -FROM rust:1.49.0-buster AS builder +FROM rust:1.51.0-buster AS builder RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl diff --git a/dev/docker/rust.dockerfile b/dev/docker/rust.dockerfile index b0e00908c9f5..19dd4879eab6 100644 --- a/dev/docker/rust.dockerfile +++ b/dev/docker/rust.dockerfile @@ -22,14 +22,21 @@ # as a mounted directory. ARG RELEASE_FLAG=--release -FROM ballistacompute/rust-base:0.4.0-20210213 AS base +FROM ballistacompute/rust-base:0.4.2-SNAPSHOT AS base WORKDIR /tmp/ballista RUN apt-get -y install cmake RUN cargo install cargo-chef FROM base as planner -ADD datafusion . -ADD ballista . +RUN mkdir /tmp/ballista/ballista +RUN mkdir /tmp/ballista/benchmarks +RUN mkdir /tmp/ballista/datafusion +RUN mkdir /tmp/ballista/datafusion-examples +ADD Cargo.toml . +COPY benchmarks ./benchmarks/ +COPY datafusion ./datafusion/ +COPY datafusion-examples ./datafusion-examples/ +COPY ballista ./ballista/ RUN cargo chef prepare --recipe-path recipe.json FROM base as cacher @@ -37,26 +44,35 @@ COPY --from=planner /tmp/ballista/recipe.json recipe.json RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json FROM base as builder -ADD datafusion . -ADD ballista . +RUN mkdir /tmp/ballista/ballista +RUN mkdir /tmp/ballista/benchmarks +RUN mkdir /tmp/ballista/datafusion +RUN mkdir /tmp/ballista/datafusion-examples +ADD Cargo.toml . +COPY benchmarks ./benchmarks/ +COPY datafusion ./datafusion/ +COPY ballista ./ballista/ +COPY datafusion-examples ./datafusion-examples/ COPY --from=cacher /tmp/ballista/target target ARG RELEASE_FLAG=--release # force build.rs to run to generate configure_me code. ENV FORCE_REBUILD='true' RUN cargo build $RELEASE_FLAG +RUN cd ballista/rust && \ + cargo build $RELEASE_FLAG # put the executor on /executor (need to be copied from different places depending on FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-executor /executor; else mv /tmp/ballista/ballista/rust/target/release/ballista-executor /executor; fi # put the scheduler on /scheduler (need to be copied from different places depending on FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/ballista/rust/target/release/ballista-scheduler /scheduler; fi # put the tpch on /tpch (need to be copied from different places depending on FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/tpch /tpch; else mv /tmp/ballista/ballista/rust/target/release/tpch /tpch; fi # Copy the binary into a new container for a smaller docker image FROM ballistacompute/rust-base:0.4.0-20210213 diff --git a/dev/integration-tests.sh b/dev/integration-tests.sh index 9ea6b314fe8b..6ed764ecda8a 100755 --- a/dev/integration-tests.sh +++ b/dev/integration-tests.sh @@ -17,6 +17,7 @@ # specific language governing permissions and limitations # under the License. set -e +./dev/build-rust-base.sh ./dev/build-rust.sh pushd ballista/rust/benchmarks/tpch ./tpch-gen.sh