Skip to content

Commit 34e1d3c

Browse files
ezyangezyang
authored
0.3.0 packaging (#221)
* Rewrite our packaging from scratch (#217) * New entry points are packaging/build_wheel.sh and packaging/build_conda.sh. The only mandatory environment variable you have to set is PYTHON_VERSION * CircleCI configuration uses 2.1-style parametrized builds to let you toggle python version, etc. as you do builds. We create a separate job per build configuration for maximum parallelism * build_tools/packaging got moved to packaging, to be in-line with directory structure in torchvision * The build_conda.sh and build_wheel.sh delegate most of the heavy lifting to pkg_helpers.bash, which defines a number of bash functions for performing common operations. The intent is that I'll copy-paste this file between other domain API projects. * TORCHAUDIO_ prefix removed from envvars, so that I can more easily share packaging scripts between projects * BUILD_VERSION is completely gone; just change the version number if you need to rebuild * No more logic for cloning and checking out a fresh copy of torchaudio * Setup cuda_suffix (#218) Signed-off-by: Edward Z. Yang <ezyang@fb.com> * Add uploading support (#220) Also bugfixes and updates to the packaging scripts. * Finish up and set parameters Signed-off-by: Edward Z. Yang <ezyang@fb.com> * Updates from vision Signed-off-by: Edward Z. Yang <ezyang@fb.com> * Go time Signed-off-by: Edward Z. Yang <ezyang@fb.com>
1 parent b2c73b6 commit 34e1d3c

File tree

14 files changed

+811
-183
lines changed

14 files changed

+811
-183
lines changed

‎.circleci/config.yml

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
version: 2.1
2+
3+
# How to test the Linux jobs:
4+
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
5+
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
6+
# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
7+
# Job names are 'name:' key.
8+
9+
binary_common: &binary_common
10+
parameters:
11+
# Edit these defaults to do a release
12+
build_version:
13+
description: "version number of release binary; by default, build a nightly"
14+
type: string
15+
default: "0.3.0"
16+
pytorch_version:
17+
description: "PyTorch version to build against; by default, use a nightly"
18+
type: string
19+
default: "1.2.0"
20+
# Don't edit these
21+
python_version:
22+
description: "Python version to build against (e.g., 3.7)"
23+
type: string
24+
unicode_abi:
25+
description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
26+
type: string
27+
default: ""
28+
environment:
29+
PYTHON_VERSION: << parameters.python_version >>
30+
BUILD_VERSION: << parameters.build_version >>
31+
PYTORCH_VERSION: << parameters.pytorch_version >>
32+
UNICODE_ABI: << parameters.unicode_abi >>
33+
CU_VERSION: cpu
34+
35+
jobs:
36+
circleci_consistency:
37+
docker:
38+
- image: circleci/python:3.7
39+
steps:
40+
- checkout
41+
- run:
42+
command: |
43+
pip install --user --progress-bar off jinja2
44+
python .circleci/regenerate.py
45+
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
46+
47+
binary_linux_wheel:
48+
<<: *binary_common
49+
docker:
50+
- image: "soumith/manylinux-cuda100"
51+
resource_class: 2xlarge+
52+
steps:
53+
- checkout
54+
- run: packaging/build_wheel.sh
55+
- store_artifacts:
56+
path: dist
57+
- persist_to_workspace:
58+
root: dist
59+
paths:
60+
- "*"
61+
62+
binary_linux_conda:
63+
<<: *binary_common
64+
docker:
65+
- image: "soumith/conda-cuda"
66+
resource_class: 2xlarge+
67+
steps:
68+
- checkout
69+
- run: packaging/build_conda.sh
70+
- store_artifacts:
71+
path: /opt/conda/conda-bld/linux-64
72+
- persist_to_workspace:
73+
root: /opt/conda/conda-bld/linux-64
74+
paths:
75+
- "*"
76+
77+
binary_macos_wheel:
78+
<<: *binary_common
79+
macos:
80+
xcode: "9.0"
81+
steps:
82+
- checkout
83+
- run:
84+
# Cannot easily deduplicate this as source'ing activate
85+
# will set environment variables which we need to propagate
86+
# to build_wheel.sh
87+
command: |
88+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
89+
sh conda.sh -b
90+
source $HOME/miniconda3/bin/activate
91+
packaging/build_wheel.sh
92+
- store_artifacts:
93+
path: dist
94+
- persist_to_workspace:
95+
root: dist
96+
paths:
97+
- "*"
98+
99+
binary_macos_conda:
100+
<<: *binary_common
101+
macos:
102+
xcode: "9.0"
103+
steps:
104+
- checkout
105+
- run:
106+
command: |
107+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
108+
sh conda.sh -b
109+
source $HOME/miniconda3/bin/activate
110+
conda install -yq conda-build
111+
packaging/build_conda.sh
112+
- store_artifacts:
113+
path: /Users/distiller/miniconda3/conda-bld/osx-64
114+
- persist_to_workspace:
115+
root: /Users/distiller/miniconda3/conda-bld/osx-64
116+
paths:
117+
- "*"
118+
119+
# Requires org-member context
120+
binary_conda_upload:
121+
docker:
122+
- image: continuumio/miniconda
123+
steps:
124+
- attach_workspace:
125+
at: ~/workspace
126+
- run:
127+
command: |
128+
# Prevent credential from leaking
129+
conda install -yq anaconda-client
130+
set +x
131+
anaconda login \
132+
--username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
133+
--password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
134+
set -x
135+
anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
136+
137+
# Requires org-member context
138+
binary_wheel_upload:
139+
docker:
140+
- image: circleci/python:3.7
141+
steps:
142+
- attach_workspace:
143+
at: ~/workspace
144+
- checkout
145+
- run:
146+
command: |
147+
pip install --user awscli
148+
export PATH="$HOME/.local/bin:$PATH"
149+
# Prevent credential from leaking
150+
set +x
151+
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
152+
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
153+
set -x
154+
for pkg in ~/workspace/*.whl; do
155+
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
156+
done
157+
158+
workflows:
159+
build:
160+
jobs:
161+
- circleci_consistency
162+
163+
- binary_linux_wheel:
164+
name: nightly_binary_linux_wheel_py2.7
165+
python_version: "2.7"
166+
- binary_wheel_upload:
167+
name: nightly_binary_linux_wheel_py2.7_upload
168+
context: org-member
169+
requires:
170+
- nightly_binary_linux_wheel_py2.7
171+
- binary_linux_wheel:
172+
name: nightly_binary_linux_wheel_py2.7_unicode
173+
python_version: "2.7"
174+
unicode_abi: "1"
175+
- binary_wheel_upload:
176+
name: nightly_binary_linux_wheel_py2.7_unicode_upload
177+
context: org-member
178+
requires:
179+
- nightly_binary_linux_wheel_py2.7_unicode
180+
- binary_linux_wheel:
181+
name: nightly_binary_linux_wheel_py3.5
182+
python_version: "3.5"
183+
- binary_wheel_upload:
184+
name: nightly_binary_linux_wheel_py3.5_upload
185+
context: org-member
186+
requires:
187+
- nightly_binary_linux_wheel_py3.5
188+
- binary_linux_wheel:
189+
name: nightly_binary_linux_wheel_py3.6
190+
python_version: "3.6"
191+
- binary_wheel_upload:
192+
name: nightly_binary_linux_wheel_py3.6_upload
193+
context: org-member
194+
requires:
195+
- nightly_binary_linux_wheel_py3.6
196+
- binary_linux_wheel:
197+
name: nightly_binary_linux_wheel_py3.7
198+
python_version: "3.7"
199+
- binary_wheel_upload:
200+
name: nightly_binary_linux_wheel_py3.7_upload
201+
context: org-member
202+
requires:
203+
- nightly_binary_linux_wheel_py3.7
204+
- binary_macos_wheel:
205+
name: nightly_binary_macos_wheel_py2.7
206+
python_version: "2.7"
207+
- binary_wheel_upload:
208+
name: nightly_binary_macos_wheel_py2.7_upload
209+
context: org-member
210+
requires:
211+
- nightly_binary_macos_wheel_py2.7
212+
- binary_macos_wheel:
213+
name: nightly_binary_macos_wheel_py2.7_unicode
214+
python_version: "2.7"
215+
unicode_abi: "1"
216+
- binary_wheel_upload:
217+
name: nightly_binary_macos_wheel_py2.7_unicode_upload
218+
context: org-member
219+
requires:
220+
- nightly_binary_macos_wheel_py2.7_unicode
221+
- binary_macos_wheel:
222+
name: nightly_binary_macos_wheel_py3.5
223+
python_version: "3.5"
224+
- binary_wheel_upload:
225+
name: nightly_binary_macos_wheel_py3.5_upload
226+
context: org-member
227+
requires:
228+
- nightly_binary_macos_wheel_py3.5
229+
- binary_macos_wheel:
230+
name: nightly_binary_macos_wheel_py3.6
231+
python_version: "3.6"
232+
- binary_wheel_upload:
233+
name: nightly_binary_macos_wheel_py3.6_upload
234+
context: org-member
235+
requires:
236+
- nightly_binary_macos_wheel_py3.6
237+
- binary_macos_wheel:
238+
name: nightly_binary_macos_wheel_py3.7
239+
python_version: "3.7"
240+
- binary_wheel_upload:
241+
name: nightly_binary_macos_wheel_py3.7_upload
242+
context: org-member
243+
requires:
244+
- nightly_binary_macos_wheel_py3.7
245+
- binary_linux_conda:
246+
name: nightly_binary_linux_conda_py2.7
247+
python_version: "2.7"
248+
- binary_conda_upload:
249+
name: nightly_binary_linux_conda_py2.7_upload
250+
context: org-member
251+
requires:
252+
- nightly_binary_linux_conda_py2.7
253+
- binary_linux_conda:
254+
name: nightly_binary_linux_conda_py3.5
255+
python_version: "3.5"
256+
- binary_conda_upload:
257+
name: nightly_binary_linux_conda_py3.5_upload
258+
context: org-member
259+
requires:
260+
- nightly_binary_linux_conda_py3.5
261+
- binary_linux_conda:
262+
name: nightly_binary_linux_conda_py3.6
263+
python_version: "3.6"
264+
- binary_conda_upload:
265+
name: nightly_binary_linux_conda_py3.6_upload
266+
context: org-member
267+
requires:
268+
- nightly_binary_linux_conda_py3.6
269+
- binary_linux_conda:
270+
name: nightly_binary_linux_conda_py3.7
271+
python_version: "3.7"
272+
- binary_conda_upload:
273+
name: nightly_binary_linux_conda_py3.7_upload
274+
context: org-member
275+
requires:
276+
- nightly_binary_linux_conda_py3.7
277+
- binary_macos_conda:
278+
name: nightly_binary_macos_conda_py2.7
279+
python_version: "2.7"
280+
- binary_conda_upload:
281+
name: nightly_binary_macos_conda_py2.7_upload
282+
context: org-member
283+
requires:
284+
- nightly_binary_macos_conda_py2.7
285+
- binary_macos_conda:
286+
name: nightly_binary_macos_conda_py3.5
287+
python_version: "3.5"
288+
- binary_conda_upload:
289+
name: nightly_binary_macos_conda_py3.5_upload
290+
context: org-member
291+
requires:
292+
- nightly_binary_macos_conda_py3.5
293+
- binary_macos_conda:
294+
name: nightly_binary_macos_conda_py3.6
295+
python_version: "3.6"
296+
- binary_conda_upload:
297+
name: nightly_binary_macos_conda_py3.6_upload
298+
context: org-member
299+
requires:
300+
- nightly_binary_macos_conda_py3.6
301+
- binary_macos_conda:
302+
name: nightly_binary_macos_conda_py3.7
303+
python_version: "3.7"
304+
- binary_conda_upload:
305+
name: nightly_binary_macos_conda_py3.7_upload
306+
context: org-member
307+
requires:
308+
- nightly_binary_macos_conda_py3.7

0 commit comments

Comments
 (0)