Skip to content

Commit

Permalink
ARROW-4909: [CI] Use hadolint to lint Dockerfiles
Browse files Browse the repository at this point in the history
Author: Krisztián Szűcs <szucs.krisztian@gmail.com>
Author: Uwe L. Korn <uwelk@xhochy.com>

Closes apache#3944 from xhochy/ARROW-4909 and squashes the following commits:

74ed305 <Krisztián Szűcs> name pre commit hooks
c97b24e <Uwe L. Korn> ARROW-4909:  Use hadolint to lint Dockerfiles
kszucs committed Mar 18, 2019
1 parent 686c44c commit 4107529
Showing 25 changed files with 189 additions and 99 deletions.
21 changes: 21 additions & 0 deletions .hadolint.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

ignored:
- DL3008
- DL3013
- DL3018
15 changes: 12 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -24,34 +24,43 @@ repos:
- repo: local
hooks:
- id: rat
name: rat
name: Release Audit Tool
language: system
entry: bash -c "git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar && ./dev/release/run-rat.sh arrow-src.tar"
always_run: true
pass_filenames: false
- id: rustfmt
name: rustfmt
name: Rust Format
language: system
entry: bash -c "cd rust && cargo +stable fmt --all -- --check"
files: ^rust/.*\.rs$
types:
- file
- rust
- id: cmake-format
name: cmake-format
name: CMake Format
language: python
entry: bash -c "pip install cmake-format && python run-cmake-format.py --check"
entry: echo
files: ^(.*/CMakeLists.txt|.*.cmake)$
- id: hadolint
name: Docker Format
language: docker_image
types:
- dockerfile
entry: --entrypoint /bin/hadolint hadolint/hadolint:latest -
exclude: ^dev/.*$
- repo: git://github.com/pre-commit/pre-commit-hooks
sha: v1.2.3
hooks:
- id: flake8
name: Python Format
files: ^(python|crossbow|integration)/
types:
- file
- python
- id: flake8
name: Cython Format
files: ^python/
types:
- file
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ before_install:
matrix:
fast_finish: true
include:
- name: "Lint C++, Python, R"
- name: "Lint C++, Python, R, Docker"
os: linux
language: python
python: "3.6"
19 changes: 12 additions & 7 deletions c_glib/Dockerfile
Original file line number Diff line number Diff line change
@@ -17,14 +17,19 @@

FROM arrow:cpp

RUN apt-get -q install --no-install-recommends -y \
ruby-dev \
pkg-config \
RUN apt-get update -y -q && \
apt-get -q install --no-install-recommends -y \
autoconf-archive \
gobject-introspection \
gtk-doc-tools \
libgirepository1.0-dev
libgirepository1.0-dev \
libglib2.0-dev \
pkg-config \
ruby-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

ADD c_glib/Gemfile /arrow/c_glib/
COPY c_glib/Gemfile /arrow/c_glib/
RUN conda install meson=0.47.1 && \
conda clean --all && \
gem install bundler && \
@@ -39,6 +44,6 @@ ENV ARROW_BUILD_TESTS=OFF \
GI_TYPELIB_PATH="${CONDA_PREFIX}/lib/girepository-1.0"

# build, install and test
CMD arrow/ci/docker_build_cpp.sh && \
CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
arrow/ci/docker_build_c_glib.sh && \
arrow/c_glib/test/run-test.rb
arrow/c_glib/test/run-test.rb"]
24 changes: 24 additions & 0 deletions ci/docker_build_and_test_cpp.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

/arrow/ci/docker_build_cpp.sh
pushd /build/cpp
ninja test
popd
23 changes: 23 additions & 0 deletions ci/docker_build_and_test_python.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

/arrow/ci/docker_build_cpp.sh
/arrow/ci/docker_build_python.sh
pytest -v --pyargs pyarrow
7 changes: 7 additions & 0 deletions ci/travis_lint.sh
Original file line number Diff line number Diff line change
@@ -23,6 +23,13 @@ set -ex
export ARROW_TRAVIS_USE_TOOLCHAIN=0
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh

pip install pre_commit
pre-commit install

# TODO: Move more checks into pre-commit as this gives a nice summary
# and doesn't abort on the first failed check.
pre-commit run hadolint -a

# CMake formatting check
pip install cmake_format
$TRAVIS_BUILD_DIR/run-cmake-format.py --check
15 changes: 8 additions & 7 deletions cpp/Dockerfile
Original file line number Diff line number Diff line change
@@ -29,16 +29,18 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
ninja-build \
pkg-config \
tzdata \
wget
wget \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

# install conda and required packages
ARG EXTRA_CONDA_PKGS
ENV PATH=/opt/conda/bin:$PATH \
CONDA_PREFIX=/opt/conda
ADD ci/docker_install_conda.sh \
ci/conda_env_cpp.yml \
ci/conda_env_unix.yml \
/arrow/ci/
COPY ci/docker_install_conda.sh \
ci/conda_env_cpp.yml \
ci/conda_env_unix.yml \
/arrow/ci/
RUN arrow/ci/docker_install_conda.sh && \
conda install -q -c conda-forge \
--file arrow/ci/conda_env_cpp.yml \
@@ -55,5 +57,4 @@ ENV CC=gcc \
PARQUET_HOME=$CONDA_PREFIX

# build and test
CMD arrow/ci/docker_build_cpp.sh && \
cd /build/cpp && ninja test
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
8 changes: 4 additions & 4 deletions cpp/Dockerfile.alpine
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.

FROM alpine
FROM alpine:3.9

# Install basic build dependencies
# grpc requires libnsl-dev to be present, this cannot be installed via a bundle.
@@ -41,14 +41,14 @@ RUN apk add --no-cache -q \

# Ganidva is deactivated as we don't support building LLVM via ExternalProject
# and Alpine only has LLVM 6 in its repositories yet.
# ARROW-4917: ORC fails with compiler problems
ENV CC=gcc \
CXX=g++ \
ARROW_GANDIVA=OFF \
ARROW_ORC=ON \
ARROW_ORC=OFF \
ARROW_PARQUET=ON \
ARROW_BUILD_TESTS=ON \
ARROW_HOME=/usr/local

# build and test
CMD arrow/ci/docker_build_cpp.sh && \
cd /build/cpp && ninja test
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
8 changes: 4 additions & 4 deletions cpp/Dockerfile.debian-testing
Original file line number Diff line number Diff line change
@@ -57,7 +57,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
rapidjson-dev \
thrift-compiler \
tzdata \
wget
wget \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

ENV CC=gcc \
CXX=g++ \
@@ -71,6 +73,4 @@ ENV CC=gcc \
CMAKE_ARGS="-DFlatbuffers_SOURCE=BUNDLED -Dc-ares_SOURCE=BUNDLED -DgRPC_SOURCE=BUNDLED"

# build and test
CMD arrow/ci/docker_build_cpp.sh && \
cd /build/cpp && ctest -j2 --output-on-failure -L unittest

CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
2 changes: 1 addition & 1 deletion cpp/Dockerfile.fedora
Original file line number Diff line number Diff line change
@@ -73,4 +73,4 @@ ENV CC=gcc \
CMAKE_ARGS="-DgRPC_SOURCE=BUNDLED"

# build; tests don't work on Fedora at the moment due to compiler problems in gcc 8.2 (fixed in 8.3)
CMD arrow/ci/docker_build_cpp.sh
CMD ["arrow/ci/docker_build_cpp.sh"]
22 changes: 11 additions & 11 deletions cpp/Dockerfile.ubuntu-bionic
Original file line number Diff line number Diff line change
@@ -17,13 +17,13 @@

FROM ubuntu:bionic

# install build essentials
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update -y -q && \
apt-get install -y wget software-properties-common
RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
RUN apt-add-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main"
RUN export DEBIAN_FRONTEND=noninteractive && \
SHELL ["/bin/bash", "-o", "pipefail", "-c"]

ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends wget software-properties-common gpg-agent && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
apt-add-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
autoconf \
@@ -60,7 +60,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
protobuf-compiler \
protobuf-compiler-grpc \
rapidjson-dev \
tzdata
tzdata \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

# Ubuntu's gtest just provides sources, the compiled version is only available
# from Ubuntu Cosmic on.
@@ -83,6 +85,4 @@ ENV CC=gcc \
CMAKE_ARGS="-DThrift_SOURCE=BUNDLED -DFlatbuffers_SOURCE=BUNDLED -DGTest_SOURCE=BUNDLED -DgRPC_SOURCE=BUNDLED -Dc-ares_SOURCE=BUNDLED -DProtobuf_SOURCE=BUNDLED"

# build and test
CMD arrow/ci/docker_build_cpp.sh && \
cd /build/cpp && ctest -j2 --output-on-failure -L unittest

CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
22 changes: 12 additions & 10 deletions cpp/Dockerfile.ubuntu-xenial
Original file line number Diff line number Diff line change
@@ -17,13 +17,13 @@

FROM ubuntu:xenial

# install build essentials
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update -y -q && \
apt-get install -y wget software-properties-common
RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
RUN apt-add-repository -y "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
RUN export DEBIAN_FRONTEND=noninteractive && \
SHELL ["/bin/bash", "-o", "pipefail", "-c"]

ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends wget software-properties-common && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
apt-add-repository -y "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main" && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
autoconf \
@@ -44,6 +44,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libgflags-dev \
libgoogle-glog-dev \
liblz4-dev \
libprotoc-dev \
libprotobuf-dev \
libre2-dev \
libsnappy-dev \
@@ -54,7 +55,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
ninja-build \
pkg-config \
protobuf-compiler \
tzdata
tzdata \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

# Benchmark is deactivated as the external project requires CMake 3.6+
# Flight is deactivated as Ubuntu provides gflags but not the CMake config for it.
@@ -74,5 +77,4 @@ ENV CC=gcc \
CMAKE_ARGS="-DThrift_SOURCE=BUNDLED -DGTest_SOURCE=BUNDLED -DFlatbuffers_SOURCE=BUNDLED -DRapidJSON_SOURCE=BUNDLED"

# build and test
CMD arrow/ci/docker_build_cpp.sh && \
cd /build/cpp && ctest -j2 --output-on-failure -L unittest
CMD ["arrow/ci/docker_build_and_test_cpp.sh"]
6 changes: 3 additions & 3 deletions csharp/build/docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -15,9 +15,9 @@
# specific language governing permissions and limitations
# under the License.

FROM microsoft/dotnet/2.1-sdk
FROM mcr.microsoft.com/dotnet/core/sdk:2.1

ADD csharp /arrow/csharp
COPY csharp /arrow/csharp
WORKDIR /arrow/csharp

CMD dotnet build
CMD ["dotnet", "build"]
6 changes: 2 additions & 4 deletions docs/Dockerfile
Original file line number Diff line number Diff line change
@@ -17,10 +17,8 @@

FROM arrow:python-3.6

ADD ci/conda_env_sphinx.yml /arrow/ci/
COPY ci/conda_env_sphinx.yml /arrow/ci/
RUN conda install --file arrow/ci/conda_env_sphinx.yml && \
conda clean --all

CMD arrow/ci/docker_build_cpp.sh && \
arrow/ci/docker_build_python.sh && \
arrow/ci/docker_build_sphinx.sh
CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && arrow/ci/docker_build_python.sh && arrow/ci/docker_build_sphinx.sh"]
11 changes: 5 additions & 6 deletions go/Dockerfile
Original file line number Diff line number Diff line change
@@ -15,14 +15,13 @@
# specific language governing permissions and limitations
# under the License.

FROM golang
FROM golang:1.12

ADD go/arrow/Gopkg.lock \
go/arrow/Gopkg.toml \
/arrow/go/arrow/
COPY go/arrow/Gopkg.lock \
go/arrow/Gopkg.toml \
/arrow/go/arrow/
WORKDIR /arrow/go/arrow

RUN go get -d -t -v ./...

CMD go install -v ./... && \
for d in $(go list ./... | grep -v vendor); do go test $d; done
CMD ["/bin/bash", "-c", "go install -v ./... && for d in $(go list ./... | grep -v vendor); do go test $d; done"]
Loading

0 comments on commit 4107529

Please sign in to comment.