Skip to content

Commit 278f8b0

Browse files
xhochyxhochy
committed
ARROW-191: Python: Provide infrastructure for manylinux1 wheels
Change-Id: I5ee41524982f7f136ca718ed781f8608f0ffb2db
1 parent bf749f5 commit 278f8b0

File tree

5 files changed

+206
-0
lines changed

5 files changed

+206
-0
lines changed

‎NOTICE.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,6 @@ This product includes software from the CMake project
3838
* Copyright 2001-2009 Kitware, Inc.
3939
* Copyright 2012-2014 Continuum Analytics, Inc.
4040
* All rights reserved.
41+
42+
This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
43+
* Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.

‎python/manylinux1/Dockerfile-parquet_arrow-base-x86_64‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License. See accompanying LICENSE file.
12+
13+
FROM arrow-base-x86_64
14+
15+
WORKDIR /
16+
ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz
17+
RUN tar xf zlib-1.2.8.tar.gz
18+
WORKDIR zlib-1.2.8
19+
RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release .
20+
RUN make -j5 install
21+
22+
WORKDIR /
23+
ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz
24+
RUN tar xf snappy-1.1.3.tar.gz
25+
WORKDIR /snappy-1.1.3
26+
RUN ./configure --with-pic --prefix=/usr
27+
RUN make -j5 install
28+
29+
WORKDIR /
30+
ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz
31+
RUN tar xf thrift-0.9.1.tar.gz
32+
WORKDIR /thrift-0.9.1
33+
RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --without-tests --with-cpp --prefix=/usr --disable-shared --enable-static
34+
RUN make -j5 install
35+
36+
WORKDIR /
37+
RUN git clone https://github.com/apache/parquet-cpp.git
38+
WORKDIR /parquet-cpp
39+
RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
40+
RUN make -j5 install

‎python/manylinux1/Dockerfile-x86_64‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License. See accompanying LICENSE file.
12+
13+
FROM quay.io/pypa/manylinux1_x86_64:latest
14+
15+
# Install dependencies
16+
RUN yum install -y flex openssl-devel
17+
18+
WORKDIR /
19+
ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz
20+
RUN tar xf boost_1_60_0.tar.gz
21+
WORKDIR /boost_1_60_0
22+
RUN ./bootstrap.sh
23+
RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
24+
25+
WORKDIR /
26+
ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz
27+
RUN tar xf cmake-3.5.2.tar.gz
28+
WORKDIR /cmake-3.5.2
29+
RUN ./configure --prefix=/usr
30+
RUN make -j5 install
31+
32+
WORKDIR /
33+
ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz
34+
RUN tar xf flatbuffers-1.3.0.tar.gz
35+
WORKDIR /flatbuffers-1.3.0
36+
RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .
37+
RUN make -j5 install
38+
39+
WORKDIR /
40+
RUN git clone https://github.com/matthew-brett/multibuild.git
41+
WORKDIR /multibuild
42+
RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
43+
44+
ADD arrow /arrow
45+
WORKDIR /arrow/cpp
46+
RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF .
47+
RUN make -j5 install

‎python/manylinux1/README.md‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<!---
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License. See accompanying LICENSE file.
13+
-->
14+
15+
## Manylinux1 wheels for Apache Arrow
16+
17+
This folder provides base Docker images and an infrastructure to build
18+
`manylinux1` compatible Python wheels that should be installable on all
19+
Linux distributions published in last four years.
20+
21+
The process is split up in two parts: There are base Docker images that build
22+
the native, Python-indenpendent dependencies. For these you can select if you
23+
want to also build the dependencies used for the Parquet support. Depending on
24+
these images, there is also a bash script that will build the pyarrow wheels
25+
for all supported Python versions and place them in the `dist` folder.
26+
27+
### Build instructions
28+
29+
```bash
30+
# Create a clean copy of the arrow source tree
31+
git clone ../../ arrow
32+
# Build the native baseimage
33+
docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
34+
# (optionally) build parquet-cpp
35+
docker build -t parquet_arrow-base-x86_64 -f Dockerfile-parquet_arrow-base-x86_64 .
36+
# Build the python packages
37+
docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
38+
# Now the new packages are located in the dist/ folder
39+
ls -l dist/
40+
```

‎python/manylinux1/build_arrow.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/bash
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License. See accompanying LICENSE file.
14+
#
15+
# Usage:
16+
# docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
17+
# or with Parquet support
18+
# docker run --rm -v $PWD:/io parquet_arrow-base-x86_64 /io/build_arrow.sh
19+
20+
# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
21+
# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
22+
23+
PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5}"
24+
25+
# Package index with only manylinux1 builds
26+
MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
27+
28+
source /multibuild/manylinux_utils.sh
29+
30+
cd /arrow/python
31+
32+
# PyArrow build configuration
33+
export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release'
34+
# Need as otherwise arrow_io is sometimes not linked
35+
export LDFLAGS="-Wl,--no-as-needed"
36+
export ARROW_HOME="/usr"
37+
38+
# Ensure the target directory exists
39+
mkdir -p /io/dist
40+
# Temporary directory to store the wheels that should be sent through auditwheel
41+
rm_mkdir unfixed_wheels
42+
43+
PY35_BIN=/opt/python/cp35-cp35m/bin
44+
$PY35_BIN/pip install 'pyelftools<0.24'
45+
$PY35_BIN/pip install 'git+https://github.com/xhochy/auditwheel.git@pyarrow-fixes'
46+
47+
# Override repair_wheelhouse function
48+
function repair_wheelhouse {
49+
local in_dir=$1
50+
local out_dir=$2
51+
for whl in $in_dir/*.whl; do
52+
if [[ $whl == *none-any.whl ]]; then
53+
cp $whl $out_dir
54+
else
55+
# Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage.
56+
auditwheel -v repair -L . $whl -w $out_dir/
57+
fi
58+
done
59+
chmod -R a+rwX $out_dir
60+
}
61+
62+
for PYTHON in ${PYTHON_VERSIONS}; do
63+
PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
64+
PIP="$(cpython_path $PYTHON)/bin/pip"
65+
PIPI_IO="$PIP install -f $MANYLINUX_URL"
66+
PATH="$PATH:$(cpython_path $PYTHON)"
67+
68+
$PIPI_IO "numpy==1.9.0"
69+
$PIPI_IO "cython==0.24"
70+
71+
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
72+
73+
rm_mkdir fixed_wheels
74+
repair_wheelhouse dist /io/dist
75+
done
76+

0 commit comments

Comments
 (0)