-
Notifications
You must be signed in to change notification settings - Fork 654
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Rewrite our packaging from scratch (#217)
* New entry points are packaging/build_wheel.sh and packaging/build_conda.sh. The only mandatory environment variable you have to set is PYTHON_VERSION * CircleCI configuration uses 2.1-style parametrized builds to let you toggle python version, etc. as you do builds. We create a separate job per build configuration for maximum parallelism * build_tools/packaging got moved to packaging, to be in-line with directory structure in torchvision * The build_conda.sh and build_wheel.sh delegate most of the heavy lifting to pkg_helpers.bash, which defines a number of bash functions for performing common operations. The intent is that I'll copy-paste this file between other domain API projects. * TORCHAUDIO_ prefix removed from envvars, so that I can more easily share packaging scripts between projects * BUILD_VERSION is completely gone; just change the version number if you need to rebuild * No more logic for cloning and checking out a fresh copy of torchaudio
- Loading branch information
Showing
14 changed files
with
492 additions
and
181 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| version: 2.1 | ||
|
|
||
| # How to test the Linux jobs: | ||
| # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ | ||
| # - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel | ||
| # - Replace binary_linux_wheel with the name of the job you want to test | ||
|
|
||
| binary_common: &binary_common | ||
| parameters: | ||
| # Edit these defaults to do a release` | ||
| build_version: | ||
| description: "version number of release binary; by default, build a nightly" | ||
| type: string | ||
| default: "" | ||
| pytorch_version: | ||
| description: "PyTorch version to build against; by default, use a nightly" | ||
| type: string | ||
| default: "" | ||
| # Don't edit these | ||
| python_version: | ||
| description: "Python version to build against (e.g., 3.7)" | ||
| type: string | ||
| unicode_abi: | ||
| description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" | ||
| type: string | ||
| default: "" | ||
| environment: | ||
| PYTHON_VERSION: << parameters.python_version >> | ||
| BUILD_VERSION: << parameters.build_version >> | ||
| PYTORCH_VERSION: << parameters.pytorch_version >> | ||
| UNICODE_ABI: << parameters.unicode_abi >> | ||
| CUDA_VERSION: cpu | ||
|
|
||
| jobs: | ||
| circleci_consistency: | ||
| docker: | ||
| - image: circleci/python:3.7 | ||
| steps: | ||
| - checkout | ||
| - run: | ||
| command: | | ||
| pip install --user --progress-bar off jinja2 | ||
| python .circleci/regenerate.py | ||
| git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) | ||
| binary_linux_wheel: | ||
| <<: *binary_common | ||
| docker: | ||
| - image: "soumith/manylinux-cuda100" | ||
| resource_class: 2xlarge+ | ||
| steps: | ||
| - checkout | ||
| - run: packaging/build_wheel.sh | ||
| - store_artifacts: | ||
| path: dist | ||
|
|
||
| binary_linux_conda: | ||
| <<: *binary_common | ||
| docker: | ||
| - image: "soumith/conda-cuda" | ||
| resource_class: 2xlarge+ | ||
| steps: | ||
| - checkout | ||
| - run: packaging/build_conda.sh | ||
| - store_artifacts: | ||
| path: /opt/conda/conda-bld/linux-64 | ||
|
|
||
| binary_macos_wheel: | ||
| <<: *binary_common | ||
| macos: | ||
| xcode: "9.0" | ||
| steps: | ||
| - checkout | ||
| - run: | ||
| # Cannot easily deduplicate this as source'ing activate | ||
| # will set environment variables which we need to propagate | ||
| # to build_wheel.sh | ||
| command: | | ||
| curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | ||
| sh conda.sh -b | ||
| source $HOME/miniconda3/bin/activate | ||
| packaging/build_wheel.sh | ||
| - store_artifacts: | ||
| path: dist | ||
|
|
||
| binary_macos_conda: | ||
| <<: *binary_common | ||
| macos: | ||
| xcode: "9.0" | ||
| steps: | ||
| - checkout | ||
| - run: | ||
| command: | | ||
| curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | ||
| sh conda.sh -b | ||
| source $HOME/miniconda3/bin/activate | ||
| conda install -yq conda-build | ||
| packaging/build_conda.sh | ||
| - store_artifacts: | ||
| path: /Users/distiller/miniconda3/conda-bld/osx-64 | ||
|
|
||
| workflows: | ||
| build: | ||
| jobs: | ||
| - circleci_consistency | ||
| - binary_linux_wheel: | ||
| name: binary_linux_wheel_py2.7 | ||
| python_version: "2.7" | ||
| - binary_linux_wheel: | ||
| name: binary_linux_wheel_py2.7_unicode | ||
| python_version: "2.7" | ||
| unicode_abi: "1" | ||
| - binary_linux_wheel: | ||
| name: binary_linux_wheel_py3.5 | ||
| python_version: "3.5" | ||
| - binary_linux_wheel: | ||
| name: binary_linux_wheel_py3.6 | ||
| python_version: "3.6" | ||
| - binary_linux_wheel: | ||
| name: binary_linux_wheel_py3.7 | ||
| python_version: "3.7" | ||
| - binary_macos_wheel: | ||
| name: binary_macos_wheel_py2.7 | ||
| python_version: "2.7" | ||
| - binary_macos_wheel: | ||
| name: binary_macos_wheel_py2.7_unicode | ||
| python_version: "2.7" | ||
| unicode_abi: "1" | ||
| - binary_macos_wheel: | ||
| name: binary_macos_wheel_py3.5 | ||
| python_version: "3.5" | ||
| - binary_macos_wheel: | ||
| name: binary_macos_wheel_py3.6 | ||
| python_version: "3.6" | ||
| - binary_macos_wheel: | ||
| name: binary_macos_wheel_py3.7 | ||
| python_version: "3.7" | ||
| - binary_linux_conda: | ||
| name: binary_linux_conda_py2.7 | ||
| python_version: "2.7" | ||
| - binary_linux_conda: | ||
| name: binary_linux_conda_py3.5 | ||
| python_version: "3.5" | ||
| - binary_linux_conda: | ||
| name: binary_linux_conda_py3.6 | ||
| python_version: "3.6" | ||
| - binary_linux_conda: | ||
| name: binary_linux_conda_py3.7 | ||
| python_version: "3.7" | ||
| - binary_macos_conda: | ||
| name: binary_macos_conda_py2.7 | ||
| python_version: "2.7" | ||
| - binary_macos_conda: | ||
| name: binary_macos_conda_py3.5 | ||
| python_version: "3.5" | ||
| - binary_macos_conda: | ||
| name: binary_macos_conda_py3.6 | ||
| python_version: "3.6" | ||
| - binary_macos_conda: | ||
| name: binary_macos_conda_py3.7 | ||
| python_version: "3.7" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| version: 2.1 | ||
|
|
||
| # How to test the Linux jobs: | ||
| # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ | ||
| # - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel | ||
| # - Replace binary_linux_wheel with the name of the job you want to test | ||
|
|
||
| binary_common: &binary_common | ||
| parameters: | ||
| # Edit these defaults to do a release` | ||
| build_version: | ||
| description: "version number of release binary; by default, build a nightly" | ||
| type: string | ||
| default: "" | ||
| pytorch_version: | ||
| description: "PyTorch version to build against; by default, use a nightly" | ||
| type: string | ||
| default: "" | ||
| # Don't edit these | ||
| python_version: | ||
| description: "Python version to build against (e.g., 3.7)" | ||
| type: string | ||
| unicode_abi: | ||
| description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" | ||
| type: string | ||
| default: "" | ||
| environment: | ||
| PYTHON_VERSION: << parameters.python_version >> | ||
| BUILD_VERSION: << parameters.build_version >> | ||
| PYTORCH_VERSION: << parameters.pytorch_version >> | ||
| UNICODE_ABI: << parameters.unicode_abi >> | ||
| CUDA_VERSION: cpu | ||
|
|
||
| jobs: | ||
| circleci_consistency: | ||
| docker: | ||
| - image: circleci/python:3.7 | ||
| steps: | ||
| - checkout | ||
| - run: | ||
| command: | | ||
| pip install --user --progress-bar off jinja2 | ||
| python .circleci/regenerate.py | ||
| git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) | ||
|
|
||
| binary_linux_wheel: | ||
| <<: *binary_common | ||
| docker: | ||
| - image: "soumith/manylinux-cuda100" | ||
| resource_class: 2xlarge+ | ||
| steps: | ||
| - checkout | ||
| - run: packaging/build_wheel.sh | ||
| - store_artifacts: | ||
| path: dist | ||
|
|
||
| binary_linux_conda: | ||
| <<: *binary_common | ||
| docker: | ||
| - image: "soumith/conda-cuda" | ||
| resource_class: 2xlarge+ | ||
| steps: | ||
| - checkout | ||
| - run: packaging/build_conda.sh | ||
| - store_artifacts: | ||
| path: /opt/conda/conda-bld/linux-64 | ||
|
|
||
| binary_macos_wheel: | ||
| <<: *binary_common | ||
| macos: | ||
| xcode: "9.0" | ||
| steps: | ||
| - checkout | ||
| - run: | ||
| # Cannot easily deduplicate this as source'ing activate | ||
| # will set environment variables which we need to propagate | ||
| # to build_wheel.sh | ||
| command: | | ||
| curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | ||
| sh conda.sh -b | ||
| source $HOME/miniconda3/bin/activate | ||
| packaging/build_wheel.sh | ||
| - store_artifacts: | ||
| path: dist | ||
|
|
||
| binary_macos_conda: | ||
| <<: *binary_common | ||
| macos: | ||
| xcode: "9.0" | ||
| steps: | ||
| - checkout | ||
| - run: | ||
| command: | | ||
| curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | ||
| sh conda.sh -b | ||
| source $HOME/miniconda3/bin/activate | ||
| conda install -yq conda-build | ||
| packaging/build_conda.sh | ||
| - store_artifacts: | ||
| path: /Users/distiller/miniconda3/conda-bld/osx-64 | ||
|
|
||
| workflows: | ||
| build: | ||
| jobs: | ||
| - circleci_consistency | ||
| {%- for btype in ["wheel", "conda"] -%} | ||
| {%- for os in ["linux", "macos"] -%} | ||
| {%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %} | ||
| - binary_{{os}}_{{btype}}: | ||
| name: binary_{{os}}_{{btype}}_py{{python_version}} | ||
| python_version: "{{python_version}}" | ||
| {%- if btype == "wheel" and python_version == "2.7" %} | ||
| - binary_{{os}}_{{btype}}: | ||
| name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode | ||
| python_version: "{{python_version}}" | ||
| unicode_abi: "1" | ||
| {%- endif -%} | ||
| {%- endfor -%} | ||
| {%- endfor -%} | ||
| {%- endfor -%} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| import jinja2 | ||
| import os.path | ||
|
|
||
| d = os.path.dirname(__file__) | ||
| env = jinja2.Environment( | ||
| loader=jinja2.FileSystemLoader(d), | ||
| lstrip_blocks=True, | ||
| autoescape=False, | ||
| ) | ||
| with open(os.path.join(d, 'config.yml'), 'w') as f: | ||
| f.write(env.get_template('config.yml.in').render()) |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.