apache · xhochy · Oct 13, 2016
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -38,3 +38,6 @@ This product includes software from the CMake project
  * Copyright 2001-2009 Kitware, Inc.
  * Copyright 2012-2014 Continuum Analytics, Inc.
  * All rights reserved.
+
+This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
+ * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
@@ -0,0 +1,40 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+FROM arrow-base-x86_64
+
+WORKDIR /
+ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz
+RUN tar xf zlib-1.2.8.tar.gz
+WORKDIR zlib-1.2.8
+RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release .
+RUN make -j5 install
+
+WORKDIR /
+ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz
+RUN tar xf snappy-1.1.3.tar.gz
+WORKDIR /snappy-1.1.3
+RUN ./configure --with-pic --prefix=/usr
+RUN make -j5 install
+
+WORKDIR /
+ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz
+RUN tar xf thrift-0.9.1.tar.gz
+WORKDIR /thrift-0.9.1
+RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --without-tests --with-cpp --prefix=/usr --disable-shared --enable-static
+RUN make -j5 install
+
+WORKDIR /
+RUN git clone https://github.com/apache/parquet-cpp.git
+WORKDIR /parquet-cpp
+RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
+RUN make -j5 install
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
@@ -0,0 +1,47 @@
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+FROM quay.io/pypa/manylinux1_x86_64:latest
+
+# Install dependencies
+RUN yum install -y flex openssl-devel
+
+WORKDIR /
+ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz
+RUN tar xf boost_1_60_0.tar.gz
+WORKDIR /boost_1_60_0
+RUN ./bootstrap.sh
+RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
+
+WORKDIR /
+ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz
+RUN tar xf cmake-3.5.2.tar.gz
+WORKDIR /cmake-3.5.2
+RUN ./configure --prefix=/usr
+RUN make -j5 install
+
+WORKDIR /
+ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz
+RUN tar xf flatbuffers-1.3.0.tar.gz
+WORKDIR /flatbuffers-1.3.0
+RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .
+RUN make -j5 install
+
+WORKDIR /
+RUN git clone https://github.com/matthew-brett/multibuild.git
+WORKDIR /multibuild
+RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
+
+ADD arrow /arrow
+WORKDIR /arrow/cpp
+RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF .
+RUN make -j5 install
diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md
@@ -0,0 +1,40 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+## Manylinux1 wheels for Apache Arrow
+
+This folder provides base Docker images and an infrastructure to build
+`manylinux1` compatible Python wheels that should be installable on all
+Linux distributions published in last four years.
+
+The process is split up in two parts: There are base Docker images that build
+the native, Python-indenpendent dependencies. For these you can select if you
+want to also build the dependencies used for the Parquet support. Depending on
+these images, there is also a bash script that will build the pyarrow wheels
+for all supported Python versions and place them in the `dist` folder.
+
+### Build instructions
+
+```bash
+# Create a clean copy of the arrow source tree
+git clone ../../ arrow
+# Build the native baseimage
+docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
+# (optionally) build parquet-cpp
+docker build -t parquet_arrow-base-x86_64 -f Dockerfile-parquet_arrow-base-x86_64 .
+# Build the python packages
+docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
+# Now the new packages are located in the dist/ folder
+ls -l dist/
+```
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+# Usage:
+#   docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
+# or with Parquet support
+#   docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
+
+# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
+# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
+
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5}"
+
+# Package index with only manylinux1 builds
+MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
+
+source /multibuild/manylinux_utils.sh
+
+cd /arrow/python
+
+# PyArrow build configuration
+export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release'
+# Need as otherwise arrow_io is sometimes not linked
+export LDFLAGS="-Wl,--no-as-needed"
+export ARROW_HOME="/usr"
+
+# Ensure the target directory exists
+mkdir -p /io/dist
+# Temporary directory to store the wheels that should be sent through auditwheel
+rm_mkdir unfixed_wheels
+
+PY35_BIN=/opt/python/cp35-cp35m/bin
+$PY35_BIN/pip install 'pyelftools<0.24'
+$PY35_BIN/pip install 'git+https://github.com/xhochy/auditwheel.git@pyarrow-fixes'
+
+# Override repair_wheelhouse function
+function repair_wheelhouse {
+    local in_dir=$1
+    local out_dir=$2
+    for whl in $in_dir/*.whl; do
+        if [[ $whl == *none-any.whl ]]; then
+            cp $whl $out_dir
+        else
+            # Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage.
+            auditwheel -v repair -L . $whl -w $out_dir/
+        fi
+    done
+    chmod -R a+rwX $out_dir
+}
+
+for PYTHON in ${PYTHON_VERSIONS}; do
+    PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
+    PIP="$(cpython_path $PYTHON)/bin/pip"
+    PIPI_IO="$PIP install -f $MANYLINUX_URL"
+    PATH="$PATH:$(cpython_path $PYTHON)"
+
+    $PIPI_IO "numpy==1.9.0"
+    $PIPI_IO "cython==0.24"
+
+    PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
+
+    rm_mkdir fixed_wheels
+    repair_wheelhouse dist /io/dist
+done
+