From 45d1da5c053c987862b8057a6aab646a10ed286c Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 7 Aug 2019 00:18:45 -0400 Subject: [PATCH] Rewrite our packaging from scratch (#217) * New entry points are packaging/build_wheel.sh and packaging/build_conda.sh. The only mandatory environment variable you have to set is PYTHON_VERSION * CircleCI configuration uses 2.1-style parametrized builds to let you toggle python version, etc. as you do builds. We create a separate job per build configuration for maximum parallelism * build_tools/packaging got moved to packaging, to be in-line with directory structure in torchvision * The build_conda.sh and build_wheel.sh delegate most of the heavy lifting to pkg_helpers.bash, which defines a number of bash functions for performing common operations. The intent is that I'll copy-paste this file between other domain API projects. * TORCHAUDIO_ prefix removed from envvars, so that I can more easily share packaging scripts between projects * BUILD_VERSION is completely gone; just change the version number if you need to rebuild * No more logic for cloning and checking out a fresh copy of torchaudio --- .circleci/config.yml | 161 ++++++++++++++++++ .circleci/config.yml.in | 120 +++++++++++++ .circleci/regenerate.py | 13 ++ build_tools/packaging/conda/build_audio.sh | 48 ------ .../packaging/wheel/linux_manywheel.sh | 46 ----- build_tools/packaging/wheel/osx_wheel.sh | 48 ------ .../packaging => packaging}/README.md | 22 +-- packaging/build_conda.sh | 12 ++ .../wheel => packaging}/build_from_source.sh | 24 ++- packaging/build_wheel.sh | 14 ++ packaging/pkg_helpers.bash | 134 +++++++++++++++ .../conda => packaging}/torchaudio/build.sh | 3 +- .../conda => packaging}/torchaudio/meta.yaml | 12 +- setup.py | 16 +- 14 files changed, 492 insertions(+), 181 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .circleci/config.yml.in create mode 100755 .circleci/regenerate.py delete mode 100755 build_tools/packaging/conda/build_audio.sh delete mode 100755 build_tools/packaging/wheel/linux_manywheel.sh delete mode 100755 build_tools/packaging/wheel/osx_wheel.sh rename {build_tools/packaging => packaging}/README.md (77%) create mode 100755 packaging/build_conda.sh rename {build_tools/packaging/wheel => packaging}/build_from_source.sh (74%) create mode 100755 packaging/build_wheel.sh create mode 100644 packaging/pkg_helpers.bash rename {build_tools/packaging/conda => packaging}/torchaudio/build.sh (64%) rename {build_tools/packaging/conda => packaging}/torchaudio/meta.yaml (69%) diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..222195c24c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,161 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel +# - Replace binary_linux_wheel with the name of the job you want to test + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release` + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CUDA_VERSION: cpu + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "soumith/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_linux_conda: + <<: *binary_common + docker: + - image: "soumith/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + +workflows: + build: + jobs: + - circleci_consistency + - binary_linux_wheel: + name: binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_linux_wheel: + name: binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_linux_wheel: + name: binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_linux_wheel: + name: binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_linux_wheel: + name: binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_macos_wheel: + name: binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_macos_wheel: + name: binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_macos_wheel: + name: binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_macos_wheel: + name: binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_linux_conda: + name: binary_linux_conda_py2.7 + python_version: "2.7" + - binary_linux_conda: + name: binary_linux_conda_py3.5 + python_version: "3.5" + - binary_linux_conda: + name: binary_linux_conda_py3.6 + python_version: "3.6" + - binary_linux_conda: + name: binary_linux_conda_py3.7 + python_version: "3.7" + - binary_macos_conda: + name: binary_macos_conda_py2.7 + python_version: "2.7" + - binary_macos_conda: + name: binary_macos_conda_py3.5 + python_version: "3.5" + - binary_macos_conda: + name: binary_macos_conda_py3.6 + python_version: "3.6" + - binary_macos_conda: + name: binary_macos_conda_py3.7 + python_version: "3.7" \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in new file mode 100644 index 0000000000..47bf1ce630 --- /dev/null +++ b/.circleci/config.yml.in @@ -0,0 +1,120 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel +# - Replace binary_linux_wheel with the name of the job you want to test + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release` + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CUDA_VERSION: cpu + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "soumith/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_linux_conda: + <<: *binary_common + docker: + - image: "soumith/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + +workflows: + build: + jobs: + - circleci_consistency +{%- for btype in ["wheel", "conda"] -%} +{%- for os in ["linux", "macos"] -%} +{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %} + - binary_{{os}}_{{btype}}: + name: binary_{{os}}_{{btype}}_py{{python_version}} + python_version: "{{python_version}}" +{%- if btype == "wheel" and python_version == "2.7" %} + - binary_{{os}}_{{btype}}: + name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode + python_version: "{{python_version}}" + unicode_abi: "1" +{%- endif -%} +{%- endfor -%} +{%- endfor -%} +{%- endfor -%} diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py new file mode 100755 index 0000000000..dcc3fb2c23 --- /dev/null +++ b/.circleci/regenerate.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import jinja2 +import os.path + +d = os.path.dirname(__file__) +env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=False, +) +with open(os.path.join(d, 'config.yml'), 'w') as f: + f.write(env.get_template('config.yml.in').render()) diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh deleted file mode 100755 index 98518200d7..0000000000 --- a/build_tools/packaging/conda/build_audio.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -if [[ -x "/remote/anaconda_token" ]]; then - . /remote/anaconda_token || true -fi - -set -ex - - # Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER=1 - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -audio_rootdir="$(pwd)/torchaudio-src" - -if [[ "$BRANCH" == "" ]]; then - BRANCH=master -fi - -if [[ ! -d "$audio_rootdir" ]]; then - rm -rf "$audio_rootdir" - git clone "https://github.com/pytorch/audio" "$audio_rootdir" -b "$BRANCH" -fi - -export TORCHAUDIO_GITHUB_ROOT_DIR="$audio_rootdir" - -cd "$SOURCE_DIR" - -ANACONDA_USER=pytorch -conda config --set anaconda_upload no - -export CONDA_CUDATOOLKIT_CONSTRAINT="" -export CUDA_VERSION="None" -if [[ "$OSTYPE" == "darwin"* ]]; then - export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ -fi - -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio -# Currently disabled, see https://github.com/pytorch/pytorch/issues/23738 -# time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio - - set +e diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh deleted file mode 100755 index 17be4c280c..0000000000 --- a/build_tools/packaging/wheel/linux_manywheel.sh +++ /dev/null @@ -1,46 +0,0 @@ -if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Pass cuda version" - echo "CUDA version should be cu90, cu100 or cpu" - exit 1 -fi -export CUVER="$1" # cu90 cu100 cpu - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER="1" -export OUT_DIR="/remote/$CUVER" - -cd /opt/python -DESIRED_PYTHON=(*/) -for desired_py in "${DESIRED_PYTHON[@]}"; do - python_installations+=("/opt/python/$desired_py") -done - -OLD_PATH=$PATH -cd /tmp -rm -rf audio -git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} -mkdir audio/third_party - -export PREFIX="/tmp/audio" -. /remote/wheel/build_from_source.sh - -cd /tmp/audio - -for PYDIR in "${python_installations[@]}"; do - export PATH=$PYDIR/bin:$OLD_PATH - pip install --upgrade pip - - # For true hermetic builds, you ought to be constructing the docker - # from scratch each time. But this makes things marginally safer if - # you aren't doing this. - pip uninstall -y torch || true - pip uninstall -y torch_nightly || true - - # NB: do not actually install requirements.txt; that is only needed for - # testing - pip install torch numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh deleted file mode 100755 index 9e23057861..0000000000 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ /dev/null @@ -1,48 +0,0 @@ -if [[ ":$PATH:" == *"conda"* ]]; then - echo "existing anaconda install in PATH, remove it and run script" - exit 1 -fi -# download and activate anaconda -rm -rf ~/minconda_wheel_env_tmp -wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ - chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ - ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ - rm Miniconda3-latest-MacOSX-x86_64.sh - -. ~/minconda_wheel_env_tmp/bin/activate - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER="1" -export OUT_DIR=~/torchaudio_wheels - -export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ - -# TODO remove when pytorch is good https://github.com/pytorch/pytorch/issues/20030 -brew install libomp -CURR_PATH=$(pwd) - -cd /tmp -rm -rf audio -git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} -mkdir audio/third_party - -export PREFIX="/tmp/audio" -. $CURR_PATH/build_from_source.sh - -cd /tmp/audio - -desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) -# for each python -for desired_python in "${desired_pythons[@]}" -do - # create and activate python env - env_name="env$desired_python" - conda create -yn $env_name python="$desired_python" - conda activate $env_name - - pip install torch numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/build_tools/packaging/README.md b/packaging/README.md similarity index 77% rename from build_tools/packaging/README.md rename to packaging/README.md index 6d2efc6634..f3ee62e62c 100644 --- a/build_tools/packaging/README.md +++ b/packaging/README.md @@ -5,12 +5,9 @@ ### Linux ```bash -# NB: We don't actually need CUDA; you don't have to run this on nvidia-docker -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash -pushd remote/conda -conda config --add channels pytorch -conda config --add channels conda-forge -./build_audio.sh +docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash +cd remote +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -32,11 +29,7 @@ anaconda upload -u pytorch /opt/conda/conda-bld/linux-64/torchaudio*.bz2 ```bash # create a fresh anaconda environment / install and activate it -cd packaging/conda -conda install -y conda-build anaconda-client -conda config --add channels pytorch -conda config --add channels conda-forge -./build_audio.sh +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -64,8 +57,8 @@ anaconda upload -u pytorch /Users/jamarshon/anaconda3/conda-bld/osx-64/torchaudi ```bash nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash -cd remote/wheel -./linux_manywheel.sh cpu +cd remote +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, @@ -84,8 +77,7 @@ cd ../cpu ### OSX ```bash -pushd wheel -./osx_wheel.sh +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh new file mode 100755 index 0000000000..46382eb422 --- /dev/null +++ b/packaging/build_conda.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +setup_build_version 0.4.0 +setup_macos +export SOURCE_ROOT_DIR="$PWD" +setup_conda_pytorch_constraint +conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio + diff --git a/build_tools/packaging/wheel/build_from_source.sh b/packaging/build_from_source.sh similarity index 74% rename from build_tools/packaging/wheel/build_from_source.sh rename to packaging/build_from_source.sh index eacd890852..3e16da5ede 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -1,11 +1,19 @@ -rm -rf source_code -mkdir source_code -pushd source_code +#!/bin/bash -wget -q -O sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -wget -q -O lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -wget -q -O flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -wget -q -O libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +set -ex + +# Arguments: PREFIX, specifying where to install dependencies into + +PREFIX="$1" + +rm -rf /tmp/torchaudio-deps +mkdir /tmp/torchaudio-deps +pushd /tmp/torchaudio-deps + +curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 @@ -43,7 +51,7 @@ pushd sox-14.4.2 ./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/sox" \ LDFLAGS="-L$PREFIX/third_party/lame/lib -L$PREFIX/third_party/flac/lib -L$PREFIX/third_party/mad/lib" \ CPPFLAGS="-I$PREFIX/third_party/lame/include -I$PREFIX/third_party/flac/include -I$PREFIX/third_party/mad/include" \ - --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking + --with-lame --with-flac --with-mad --without-coreaudio --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh new file mode 100755 index 0000000000..8871629c83 --- /dev/null +++ b/packaging/build_wheel.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +setup_python +setup_build_version 0.4.0 +setup_macos +"$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies +pip_install numpy future +setup_pip_pytorch_version +python setup.py clean +IS_WHEEL=1 python setup.py bdist_wheel diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash new file mode 100644 index 0000000000..0265d5bd60 --- /dev/null +++ b/packaging/pkg_helpers.bash @@ -0,0 +1,134 @@ +# A set of useful bash functions for common functionality we need to do in +# many build scripts + +# Respecting PYTHON_VERSION and UNICODE_ABI, add (or install) the correct +# version of Python to perform a build. Relevant to wheel builds. +setup_python() { + if [[ "$(uname)" == Darwin ]]; then + eval "$(conda shell.bash hook)" + conda env remove -n "env$PYTHON_VERSION" || true + conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" + conda activate "env$PYTHON_VERSION" + else + case "$PYTHON_VERSION" in + 2.7) + if [[ -n "$UNICODE_ABI" ]]; then + python_abi=cp27-cp27mu + else + python_abi=cp27-cp27m + fi + ;; + 3.5) python_abi=cp35-cp35m ;; + 3.6) python_abi=cp36-cp36m ;; + 3.7) python_abi=cp37-cp37m ;; + *) + echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + ;; + esac + export PATH="/opt/python/$python_abi/bin:$PATH" + fi +} + +# Fill CUDA_SUFFIX with CUDA_VERSION, but don't fill it for the default +# CUDA version (that's a blank suffix) +setup_cuda_suffix() { + if [[ "$(uname)" == Darwin ]]; then + if [[ "$CUDA_VERSION" != "cpu" ]]; then + echo "CUDA_VERSION on OS X must be cpu" + exit 1 + fi + export CPU_SUFFIX="" + else + case "$CUDA_VERSION" in + 10.0) + export CUDA_SUFFIX="" ;; # default! + 9.2) + export CUDA_SUFFIX="+cu92" ;; + cpu) + export CUDA_SUFFIX="+cpu" ;; + *) + echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" + esac + export CPU_SUFFIX="+cpu" + fi +} + +# If a package is cpu-only, we never provide a cuda suffix +setup_cpuonly_cuda_suffix() { + export CUDA_SUFFIX="" + export CPU_SUFFIX="" +} + +# Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string +# Usage: setup_build_version 0.2 +setup_build_version() { + if [[ -z "$BUILD_VERSION" ]]; then + export BUILD_VERSION="$1.dev$(date "+%Y%m%d")" + fi +} + +# Set some useful variables for OS X, if applicable +setup_macos() { + if [[ "$(uname)" == Darwin ]]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ + fi +} + +# Function to retry functions that sometimes timeout or have flaky failures +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +# Install with pip a bit more robustly than the default +pip_install() { + retry pip install --progress-bar off "$@" +} + +# Install torch with pip, respecting PYTORCH_VERSION, and record the installed +# version into PYTORCH_VERSION, if applicable +setup_pip_pytorch_version() { + if [[ -z "$PYTORCH_VERSION" ]]; then + # Install latest prerelease CPU version of torch, per our nightlies. + pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + # CPU/CUDA variants of PyTorch have ABI compatible PyTorch. Therefore, we + # strip off the local package qualifier. + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + else + # TODO: Maybe add staging too + pip_install "torch==$PYTORCH_VERSION" \ + -f https://download.pytorch.org/whl/torch_stable.html + fi +} + +# Fill PYTORCH_VERSION with the latest conda nightly version, and +# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions +# +# You MUST have populated CUDA_SUFFIX before hand. +# +# TODO: This is currently hard-coded for CPU-only case +setup_conda_pytorch_constraint() { + if [[ -z "$PYTORCH_VERSION" ]]; then + export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" + else + export CONDA_CHANNEL_FLAGS="-c pytorch" + fi + if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${CPU_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" + else + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" + fi +} + +# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT +setup_conda_cudatoolkit_constraint() { + if [[ "$CUDA_VERSION" == cpu ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="" + else + echo + # TODO + fi +} diff --git a/build_tools/packaging/conda/torchaudio/build.sh b/packaging/torchaudio/build.sh similarity index 64% rename from build_tools/packaging/conda/torchaudio/build.sh rename to packaging/torchaudio/build.sh index 7e5917060f..dff89cf31b 100644 --- a/build_tools/packaging/conda/torchaudio/build.sh +++ b/packaging/torchaudio/build.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -ex -PREFIX="$(pwd)" -. build_tools/packaging/wheel/build_from_source.sh +packaging/build_from_source.sh "$(pwd)" IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml similarity index 69% rename from build_tools/packaging/conda/torchaudio/meta.yaml rename to packaging/torchaudio/meta.yaml index fdf41e52a9..0b89969fa2 100644 --- a/build_tools/packaging/conda/torchaudio/meta.yaml +++ b/packaging/torchaudio/meta.yaml @@ -1,9 +1,9 @@ package: name: torchaudio - version: "{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}" + version: "{{ environ.get('BUILD_VERSION') }}" source: - path: "{{ environ.get('TORCHAUDIO_GITHUB_ROOT_DIR') }}" + path: "{{ environ.get('SOURCE_ROOT_DIR') }}" requirements: build: @@ -13,16 +13,16 @@ requirements: host: - python - setuptools - - pytorch >=1.2.0 + - cpuonly + {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} run: - python - typing - - pytorch >=1.2.0 + {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} build: - number: {{ environ.get('TORCHAUDIO_BUILD_NUMBER') }} - string: py{{py}}_{{environ.get('TORCHAUDIO_BUILD_NUMBER')}} + string: py{{py}} test: imports: diff --git a/setup.py b/setup.py index eea67ae8ec..2d2ea43860 100644 --- a/setup.py +++ b/setup.py @@ -63,12 +63,8 @@ def check_env_flag(name, default=''): except Exception: pass -if os.getenv('TORCHAUDIO_BUILD_VERSION'): - assert os.getenv('TORCHAUDIO_BUILD_NUMBER') is not None - build_number = int(os.getenv('TORCHAUDIO_BUILD_NUMBER')) - version = os.getenv('TORCHAUDIO_BUILD_VERSION') - if build_number > 1: - version += '.post' + str(build_number) +if os.getenv('BUILD_VERSION'): + version = os.getenv('BUILD_VERSION') elif sha != 'Unknown': version += '+' + sha[:7] print('-- Building version ' + version) @@ -78,11 +74,15 @@ def check_env_flag(name, default=''): f.write("__version__ = '{}'\n".format(version)) f.write("git_version = {}\n".format(repr(sha))) -pytorch_package_name = os.getenv('TORCHAUDIO_PYTORCH_DEPENDENCY_NAME', 'torch') +pytorch_package_version = os.getenv('PYTORCH_VERSION') + +pytorch_package_dep = 'torch' +if pytorch_package_version is not None: + pytorch_package_dep += "==" + pytorch_package_version setup( name="torchaudio", - version="0.2", + version=version, description="An audio package for PyTorch", url="https://github.com/pytorch/audio", author="Soumith Chintala, David Pollack, Sean Naren, Peter Goldsborough",