Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add binary build #766

Merged
merged 1 commit into from
May 13, 2020
Merged

Add binary build #766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
275 changes: 264 additions & 11 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,13 @@ jobs:
steps:
- checkout
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
# to build_wheel.sh
# Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution.
command: |
curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz
tar -xzf cmake.tar.gz
cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/
cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/
export PATH="${PATH}:/usr/local/bin"
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
Expand Down Expand Up @@ -287,13 +290,48 @@ jobs:


workflows:
# build:
# jobs:
# - circleci_consistency
#
# - binary_win_conda:
# name: torchtext_win_py3.6
# python_version: "3.6"
build:
jobs:
- circleci_consistency
- binary_linux_wheel:
name: binary_linux_wheel_py3.6
python_version: '3.6'
- binary_linux_wheel:
name: binary_linux_wheel_py3.7
python_version: '3.7'
- binary_linux_wheel:
name: binary_linux_wheel_py3.8
python_version: '3.8'
- binary_macos_wheel:
name: binary_macos_wheel_py3.6
python_version: '3.6'
- binary_macos_wheel:
name: binary_macos_wheel_py3.7
python_version: '3.7'
- binary_macos_wheel:
name: binary_macos_wheel_py3.8
python_version: '3.8'
- binary_linux_conda:
name: binary_linux_conda_py3.6
python_version: '3.6'
- binary_linux_conda:
name: binary_linux_conda_py3.7
python_version: '3.7'
- binary_linux_conda:
name: binary_linux_conda_py3.8
python_version: '3.8'
- binary_macos_conda:
name: binary_macos_conda_py3.6
python_version: '3.6'
- binary_macos_conda:
name: binary_macos_conda_py3.7
python_version: '3.7'
- binary_macos_conda:
name: binary_macos_conda_py3.8
python_version: '3.8'
# - binary_win_conda:
# name: torchtext_win_py3.6
# python_version: "3.6"
unittest:
jobs:
- unittest_linux:
Expand All @@ -311,7 +349,222 @@ workflows:
filters:
branches:
only: nightly
#
- binary_linux_wheel:
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.6
python_version: '3.6'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.6_upload
requires:
- nightly_binary_linux_wheel_py3.6
- smoke_test_linux_pip:
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.6_smoke_test_pip
python_version: '3.6'
requires:
- nightly_binary_linux_wheel_py3.6_upload
- binary_linux_wheel:
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.7
python_version: '3.7'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.7_upload
requires:
- nightly_binary_linux_wheel_py3.7
- smoke_test_linux_pip:
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.7_smoke_test_pip
python_version: '3.7'
requires:
- nightly_binary_linux_wheel_py3.7_upload
- binary_linux_wheel:
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.8
python_version: '3.8'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.8_upload
requires:
- nightly_binary_linux_wheel_py3.8
- smoke_test_linux_pip:
filters:
branches:
only: nightly
name: nightly_binary_linux_wheel_py3.8_smoke_test_pip
python_version: '3.8'
requires:
- nightly_binary_linux_wheel_py3.8_upload
- binary_macos_wheel:
filters:
branches:
only: nightly
name: nightly_binary_macos_wheel_py3.6
python_version: '3.6'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_macos_wheel_py3.6_upload
requires:
- nightly_binary_macos_wheel_py3.6
- binary_macos_wheel:
filters:
branches:
only: nightly
name: nightly_binary_macos_wheel_py3.7
python_version: '3.7'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_macos_wheel_py3.7_upload
requires:
- nightly_binary_macos_wheel_py3.7
- binary_macos_wheel:
filters:
branches:
only: nightly
name: nightly_binary_macos_wheel_py3.8
python_version: '3.8'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_macos_wheel_py3.8_upload
requires:
- nightly_binary_macos_wheel_py3.8
- binary_linux_conda:
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.6
python_version: '3.6'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.6_upload
requires:
- nightly_binary_linux_conda_py3.6
- smoke_test_linux_conda:
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.6_smoke_test_conda
python_version: '3.6'
requires:
- nightly_binary_linux_conda_py3.6_upload
- binary_linux_conda:
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.7
python_version: '3.7'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.7_upload
requires:
- nightly_binary_linux_conda_py3.7
- smoke_test_linux_conda:
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.7_smoke_test_conda
python_version: '3.7'
requires:
- nightly_binary_linux_conda_py3.7_upload
- binary_linux_conda:
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.8
python_version: '3.8'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.8_upload
requires:
- nightly_binary_linux_conda_py3.8
- smoke_test_linux_conda:
filters:
branches:
only: nightly
name: nightly_binary_linux_conda_py3.8_smoke_test_conda
python_version: '3.8'
requires:
- nightly_binary_linux_conda_py3.8_upload
- binary_macos_conda:
filters:
branches:
only: nightly
name: nightly_binary_macos_conda_py3.6
python_version: '3.6'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_macos_conda_py3.6_upload
requires:
- nightly_binary_macos_conda_py3.6
- binary_macos_conda:
filters:
branches:
only: nightly
name: nightly_binary_macos_conda_py3.7
python_version: '3.7'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_macos_conda_py3.7_upload
requires:
- nightly_binary_macos_conda_py3.7
- binary_macos_conda:
filters:
branches:
only: nightly
name: nightly_binary_macos_conda_py3.8
python_version: '3.8'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_macos_conda_py3.8_upload
requires:
- nightly_binary_macos_conda_py3.8
docker_build:
triggers:
- schedule:
Expand Down
25 changes: 14 additions & 11 deletions .circleci/config.yml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,13 @@ jobs:
steps:
- checkout
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
# to build_wheel.sh
# Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution.
command: |
curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz
tar -xzf cmake.tar.gz
cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/
cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/
export PATH="${PATH}:/usr/local/bin"
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
sh conda.sh -b
source $HOME/miniconda3/bin/activate
Expand Down Expand Up @@ -287,13 +290,13 @@ jobs:


workflows:
# build:
# jobs:
# - circleci_consistency
# {{ build_workflows() }}
# - binary_win_conda:
# name: torchtext_win_py3.6
# python_version: "3.6"
build:
jobs:
- circleci_consistency
{{ build_workflows() }}
# - binary_win_conda:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this to be fixed in a follow-up PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not my intent.
I am just keeping what was there and making build_workflows work.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we test the torchtext package with cpp extension on a Windows machine? Although we don't explicitly support Windows, the current package does work on Windows.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, we should. torchaudio does not have this but needs this soon.
We can look into how torchvision does it or ask help from someone with Windows experience.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@peterjc123

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add unit tests first. Binary builds on Windows will come later.

# name: torchtext_win_py3.6
# python_version: "3.6"
unittest:
jobs:
{{ unittest_workflows() }}
Expand All @@ -303,7 +306,7 @@ workflows:
filters:
branches:
only: nightly
# {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
{{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
docker_build:
triggers:
- schedule:
Expand Down
4 changes: 0 additions & 4 deletions .circleci/regenerate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,12 @@


def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
'''
w = []
for btype in ["wheel", "conda"]:
for os_type in ["linux", "macos"]:
for python_version in PYTHON_VERSIONS:
w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload)

return indent(indentation, w)
'''
return ''


def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,5 @@ venv.bak/
# vim
*.swp
*.swo

torchtext/version.py
12 changes: 12 additions & 0 deletions packaging/build_conda.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -ex

script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
. "$script_dir/pkg_helpers.bash"

export BUILD_TYPE="conda"
export NO_CUDA_PACKAGE=1
setup_env 0.6.0
export SOURCE_ROOT_DIR="$PWD"
setup_conda_pytorch_constraint
conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchtext
15 changes: 15 additions & 0 deletions packaging/build_wheel.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/bash
set -ex

script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
. "$script_dir/pkg_helpers.bash"

export BUILD_TYPE="wheel"
export NO_CUDA_PACKAGE=1
setup_env 0.6.0
setup_wheel_python
pip_install numpy future
setup_pip_pytorch_version
git submodule update --init --recursive
python setup.py clean
python setup.py bdist_wheel
Loading