diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 0000000000..222195c24c
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,161 @@
+version: 2.1
+
+# How to test the Linux jobs:
+#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
+#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel
+#     - Replace binary_linux_wheel with the name of the job you want to test
+
+binary_common: &binary_common
+  parameters:
+    # Edit these defaults to do a release`
+    build_version:
+      description: "version number of release binary; by default, build a nightly"
+      type: string
+      default: ""
+    pytorch_version:
+      description: "PyTorch version to build against; by default, use a nightly"
+      type: string
+      default: ""
+    # Don't edit these
+    python_version:
+      description: "Python version to build against (e.g., 3.7)"
+      type: string
+    unicode_abi:
+      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
+      type: string
+      default: ""
+  environment:
+    PYTHON_VERSION: << parameters.python_version >>
+    BUILD_VERSION: << parameters.build_version >>
+    PYTORCH_VERSION: << parameters.pytorch_version >>
+    UNICODE_ABI: << parameters.unicode_abi >>
+    CUDA_VERSION: cpu
+
+jobs:
+  circleci_consistency:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - run:
+          command: |
+            pip install --user --progress-bar off jinja2
+            python .circleci/regenerate.py
+            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
+
+  binary_linux_wheel:
+    <<: *binary_common
+    docker:
+      - image: "soumith/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: packaging/build_wheel.sh
+      - store_artifacts:
+          path: dist
+
+  binary_linux_conda:
+    <<: *binary_common
+    docker:
+      - image: "soumith/conda-cuda"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: packaging/build_conda.sh
+      - store_artifacts:
+          path: /opt/conda/conda-bld/linux-64
+
+  binary_macos_wheel:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          # Cannot easily deduplicate this as source'ing activate
+          # will set environment variables which we need to propagate
+          # to build_wheel.sh
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            packaging/build_wheel.sh
+      - store_artifacts:
+          path: dist
+
+  binary_macos_conda:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            conda install -yq conda-build
+            packaging/build_conda.sh
+      - store_artifacts:
+          path: /Users/distiller/miniconda3/conda-bld/osx-64
+
+workflows:
+  build:
+    jobs:
+      - circleci_consistency
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py2.7
+          python_version: "2.7"
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py2.7_unicode
+          python_version: "2.7"
+          unicode_abi: "1"
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py3.5
+          python_version: "3.5"
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py3.6
+          python_version: "3.6"
+      - binary_linux_wheel:
+          name: binary_linux_wheel_py3.7
+          python_version: "3.7"
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py2.7
+          python_version: "2.7"
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py2.7_unicode
+          python_version: "2.7"
+          unicode_abi: "1"
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py3.5
+          python_version: "3.5"
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py3.6
+          python_version: "3.6"
+      - binary_macos_wheel:
+          name: binary_macos_wheel_py3.7
+          python_version: "3.7"
+      - binary_linux_conda:
+          name: binary_linux_conda_py2.7
+          python_version: "2.7"
+      - binary_linux_conda:
+          name: binary_linux_conda_py3.5
+          python_version: "3.5"
+      - binary_linux_conda:
+          name: binary_linux_conda_py3.6
+          python_version: "3.6"
+      - binary_linux_conda:
+          name: binary_linux_conda_py3.7
+          python_version: "3.7"
+      - binary_macos_conda:
+          name: binary_macos_conda_py2.7
+          python_version: "2.7"
+      - binary_macos_conda:
+          name: binary_macos_conda_py3.5
+          python_version: "3.5"
+      - binary_macos_conda:
+          name: binary_macos_conda_py3.6
+          python_version: "3.6"
+      - binary_macos_conda:
+          name: binary_macos_conda_py3.7
+          python_version: "3.7"
\ No newline at end of file
diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
new file mode 100644
index 0000000000..47bf1ce630
--- /dev/null
+++ b/.circleci/config.yml.in
@@ -0,0 +1,120 @@
+version: 2.1
+
+# How to test the Linux jobs:
+#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
+#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel
+#     - Replace binary_linux_wheel with the name of the job you want to test
+
+binary_common: &binary_common
+  parameters:
+    # Edit these defaults to do a release`
+    build_version:
+      description: "version number of release binary; by default, build a nightly"
+      type: string
+      default: ""
+    pytorch_version:
+      description: "PyTorch version to build against; by default, use a nightly"
+      type: string
+      default: ""
+    # Don't edit these
+    python_version:
+      description: "Python version to build against (e.g., 3.7)"
+      type: string
+    unicode_abi:
+      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
+      type: string
+      default: ""
+  environment:
+    PYTHON_VERSION: << parameters.python_version >>
+    BUILD_VERSION: << parameters.build_version >>
+    PYTORCH_VERSION: << parameters.pytorch_version >>
+    UNICODE_ABI: << parameters.unicode_abi >>
+    CUDA_VERSION: cpu
+
+jobs:
+  circleci_consistency:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - run:
+          command: |
+            pip install --user --progress-bar off jinja2
+            python .circleci/regenerate.py
+            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
+
+  binary_linux_wheel:
+    <<: *binary_common
+    docker:
+      - image: "soumith/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: packaging/build_wheel.sh
+      - store_artifacts:
+          path: dist
+
+  binary_linux_conda:
+    <<: *binary_common
+    docker:
+      - image: "soumith/conda-cuda"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: packaging/build_conda.sh
+      - store_artifacts:
+          path: /opt/conda/conda-bld/linux-64
+
+  binary_macos_wheel:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          # Cannot easily deduplicate this as source'ing activate
+          # will set environment variables which we need to propagate
+          # to build_wheel.sh
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            packaging/build_wheel.sh
+      - store_artifacts:
+          path: dist
+
+  binary_macos_conda:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            conda install -yq conda-build
+            packaging/build_conda.sh
+      - store_artifacts:
+          path: /Users/distiller/miniconda3/conda-bld/osx-64
+
+workflows:
+  build:
+    jobs:
+      - circleci_consistency
+{%- for btype in ["wheel", "conda"] -%}
+{%-   for os in ["linux", "macos"] -%}
+{%-     for python_version in ["2.7", "3.5", "3.6", "3.7"] %}
+      - binary_{{os}}_{{btype}}:
+          name: binary_{{os}}_{{btype}}_py{{python_version}}
+          python_version: "{{python_version}}"
+{%-       if btype == "wheel" and python_version == "2.7" %}
+      - binary_{{os}}_{{btype}}:
+          name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode
+          python_version: "{{python_version}}"
+          unicode_abi: "1"
+{%-       endif -%}
+{%-     endfor -%}
+{%-   endfor -%}
+{%-   endfor -%}
diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py
new file mode 100755
index 0000000000..dcc3fb2c23
--- /dev/null
+++ b/.circleci/regenerate.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import jinja2
+import os.path
+
+d = os.path.dirname(__file__)
+env = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(d),
+    lstrip_blocks=True,
+    autoescape=False,
+)
+with open(os.path.join(d, 'config.yml'), 'w') as f:
+    f.write(env.get_template('config.yml.in').render())
diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh
deleted file mode 100755
index 98518200d7..0000000000
--- a/build_tools/packaging/conda/build_audio.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-if [[ -x "/remote/anaconda_token" ]]; then
-    . /remote/anaconda_token || true
-fi
-
-set -ex
-
- # Function to retry functions that sometimes timeout or have flaky failures
-retry () {
-    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
-}
-
-export TORCHAUDIO_BUILD_VERSION="0.3.0"
-export TORCHAUDIO_BUILD_NUMBER=1
-
-SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-audio_rootdir="$(pwd)/torchaudio-src"
-
-if [[ "$BRANCH" == "" ]]; then
-  BRANCH=master
-fi
-
-if [[ ! -d "$audio_rootdir" ]]; then
-    rm -rf "$audio_rootdir"
-    git clone "https://github.com/pytorch/audio" "$audio_rootdir" -b "$BRANCH"
-fi
-
-export TORCHAUDIO_GITHUB_ROOT_DIR="$audio_rootdir"
-
-cd "$SOURCE_DIR"
-
-ANACONDA_USER=pytorch
-conda config --set anaconda_upload no
-
-export CONDA_CUDATOOLKIT_CONSTRAINT=""
-export CUDA_VERSION="None"
-if [[ "$OSTYPE" == "darwin"* ]]; then
-  export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++
-fi
-
-time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio
-# Currently disabled, see https://github.com/pytorch/pytorch/issues/23738
-# time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio
-time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio
-time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio
-
- set +e
diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh
deleted file mode 100755
index 17be4c280c..0000000000
--- a/build_tools/packaging/wheel/linux_manywheel.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-if [ "$#" -ne 1 ]; then
-    echo "Illegal number of parameters. Pass cuda version"
-    echo "CUDA version should be cu90, cu100 or cpu"
-    exit 1
-fi
-export CUVER="$1" # cu90 cu100 cpu
-
-export TORCHAUDIO_BUILD_VERSION="0.3.0"
-export TORCHAUDIO_BUILD_NUMBER="1"
-export OUT_DIR="/remote/$CUVER"
-
-cd /opt/python
-DESIRED_PYTHON=(*/)
-for desired_py in "${DESIRED_PYTHON[@]}"; do
-    python_installations+=("/opt/python/$desired_py")
-done
-
-OLD_PATH=$PATH
-cd /tmp
-rm -rf audio
-git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION}
-mkdir audio/third_party
-
-export PREFIX="/tmp/audio"
-. /remote/wheel/build_from_source.sh
-
-cd /tmp/audio
-
-for PYDIR in "${python_installations[@]}"; do
-    export PATH=$PYDIR/bin:$OLD_PATH
-    pip install --upgrade pip
-
-    # For true hermetic builds, you ought to be constructing the docker
-    # from scratch each time.  But this makes things marginally safer if
-    # you aren't doing this.
-    pip uninstall -y torch || true
-    pip uninstall -y torch_nightly || true
-
-    # NB: do not actually install requirements.txt; that is only needed for
-    # testing
-    pip install torch numpy future
-    IS_WHEEL=1 python setup.py clean
-    IS_WHEEL=1 python setup.py bdist_wheel
-    mkdir -p $OUT_DIR
-    cp dist/*.whl $OUT_DIR/
-done
diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh
deleted file mode 100755
index 9e23057861..0000000000
--- a/build_tools/packaging/wheel/osx_wheel.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-if [[ ":$PATH:" == *"conda"* ]]; then
-    echo "existing anaconda install in PATH, remove it and run script"
-    exit 1
-fi
-# download and activate anaconda
-rm -rf ~/minconda_wheel_env_tmp
-wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \
-    chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \
-    ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \
-    rm Miniconda3-latest-MacOSX-x86_64.sh
-
-. ~/minconda_wheel_env_tmp/bin/activate
-
-export TORCHAUDIO_BUILD_VERSION="0.3.0"
-export TORCHAUDIO_BUILD_NUMBER="1"
-export OUT_DIR=~/torchaudio_wheels
-
-export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++
-
-# TODO remove when pytorch is good https://github.com/pytorch/pytorch/issues/20030
-brew install libomp
-CURR_PATH=$(pwd)
-
-cd /tmp
-rm -rf audio
-git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION}
-mkdir audio/third_party
-
-export PREFIX="/tmp/audio"
-. $CURR_PATH/build_from_source.sh
-
-cd /tmp/audio
-
-desired_pythons=( "2.7" "3.5" "3.6" "3.7" )
-# for each python
-for desired_python in "${desired_pythons[@]}"
-do
-    # create and activate python env
-    env_name="env$desired_python"
-    conda create -yn $env_name python="$desired_python"
-    conda activate $env_name
-
-    pip install torch numpy future
-    IS_WHEEL=1 python setup.py clean
-    IS_WHEEL=1 python setup.py bdist_wheel
-    mkdir -p $OUT_DIR
-    cp dist/*.whl $OUT_DIR/
-done
diff --git a/build_tools/packaging/README.md b/packaging/README.md
similarity index 77%
rename from build_tools/packaging/README.md
rename to packaging/README.md
index 6d2efc6634..f3ee62e62c 100644
--- a/build_tools/packaging/README.md
+++ b/packaging/README.md
@@ -5,12 +5,9 @@
  ### Linux
 
  ```bash
-# NB: We don't actually need CUDA; you don't have to run this on nvidia-docker
-nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash
-pushd remote/conda
-conda config --add channels pytorch
-conda config --add channels conda-forge
-./build_audio.sh
+docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash
+cd remote
+PYTHON_VERSION=3.7 packaging/build_conda.sh
 ```
 
 To install bz2,
@@ -32,11 +29,7 @@ anaconda upload -u pytorch /opt/conda/conda-bld/linux-64/torchaudio*.bz2
 
  ```bash
 # create a fresh anaconda environment / install and activate it
-cd packaging/conda
-conda install -y conda-build anaconda-client
-conda config --add channels pytorch
-conda config --add channels conda-forge
-./build_audio.sh
+PYTHON_VERSION=3.7 packaging/build_conda.sh
 ```
 
 To install bz2,
@@ -64,8 +57,8 @@ anaconda upload -u pytorch /Users/jamarshon/anaconda3/conda-bld/osx-64/torchaudi
 
  ```bash
 nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash
-cd remote/wheel
-./linux_manywheel.sh cpu
+cd remote
+PYTHON_VERSION=3.7 packaging/build_wheel.sh
 ```
 
 To install wheels,
@@ -84,8 +77,7 @@ cd ../cpu
  ### OSX
 
  ```bash
-pushd wheel
-./osx_wheel.sh
+PYTHON_VERSION=3.7 packaging/build_wheel.sh
 ```
 
 To install wheels,
diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh
new file mode 100755
index 0000000000..46382eb422
--- /dev/null
+++ b/packaging/build_conda.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. "$script_dir/pkg_helpers.bash"
+
+setup_build_version 0.4.0
+setup_macos
+export SOURCE_ROOT_DIR="$PWD"
+setup_conda_pytorch_constraint
+conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio
+
diff --git a/build_tools/packaging/wheel/build_from_source.sh b/packaging/build_from_source.sh
similarity index 74%
rename from build_tools/packaging/wheel/build_from_source.sh
rename to packaging/build_from_source.sh
index eacd890852..3e16da5ede 100755
--- a/build_tools/packaging/wheel/build_from_source.sh
+++ b/packaging/build_from_source.sh
@@ -1,11 +1,19 @@
-rm -rf source_code
-mkdir source_code
-pushd source_code
+#!/bin/bash
 
-wget -q -O sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet"
-wget -q -O lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent"
-wget -q -O flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz"
-wget -q -O libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz"
+set -ex
+
+# Arguments: PREFIX, specifying where to install dependencies into
+
+PREFIX="$1"
+
+rm -rf /tmp/torchaudio-deps
+mkdir /tmp/torchaudio-deps
+pushd /tmp/torchaudio-deps
+
+curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet"
+curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent"
+curl -L -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz"
+curl -L -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz"
 
 # unpack the dependencies
 tar xfp sox-14.4.2.tar.bz2
@@ -43,7 +51,7 @@ pushd sox-14.4.2
 ./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/sox" \
     LDFLAGS="-L$PREFIX/third_party/lame/lib -L$PREFIX/third_party/flac/lib -L$PREFIX/third_party/mad/lib" \
     CPPFLAGS="-I$PREFIX/third_party/lame/include -I$PREFIX/third_party/flac/include -I$PREFIX/third_party/mad/include" \
-    --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
+    --with-lame --with-flac --with-mad --without-coreaudio --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
 make -s -j && make install
 popd
 
diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh
new file mode 100755
index 0000000000..8871629c83
--- /dev/null
+++ b/packaging/build_wheel.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -ex
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. "$script_dir/pkg_helpers.bash"
+
+setup_python
+setup_build_version 0.4.0
+setup_macos
+"$script_dir/build_from_source.sh" "$(pwd)"  # Build static dependencies
+pip_install numpy future
+setup_pip_pytorch_version
+python setup.py clean
+IS_WHEEL=1 python setup.py bdist_wheel
diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
new file mode 100644
index 0000000000..0265d5bd60
--- /dev/null
+++ b/packaging/pkg_helpers.bash
@@ -0,0 +1,134 @@
+# A set of useful bash functions for common functionality we need to do in
+# many build scripts
+
+# Respecting PYTHON_VERSION and UNICODE_ABI, add (or install) the correct
+# version of Python to perform a build.  Relevant to wheel builds.
+setup_python() {
+  if [[ "$(uname)" == Darwin ]]; then
+    eval "$(conda shell.bash hook)"
+    conda env remove -n "env$PYTHON_VERSION" || true
+    conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
+    conda activate "env$PYTHON_VERSION"
+  else
+    case "$PYTHON_VERSION" in
+      2.7)
+        if [[ -n "$UNICODE_ABI" ]]; then
+          python_abi=cp27-cp27mu
+        else
+          python_abi=cp27-cp27m
+        fi
+        ;;
+      3.5) python_abi=cp35-cp35m ;;
+      3.6) python_abi=cp36-cp36m ;;
+      3.7) python_abi=cp37-cp37m ;;
+      *)
+        echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION"
+        exit 1
+        ;;
+    esac
+    export PATH="/opt/python/$python_abi/bin:$PATH"
+  fi
+}
+
+# Fill CUDA_SUFFIX with CUDA_VERSION, but don't fill it for the default
+# CUDA version (that's a blank suffix)
+setup_cuda_suffix() {
+  if [[ "$(uname)" == Darwin ]]; then
+    if [[ "$CUDA_VERSION" != "cpu" ]]; then
+      echo "CUDA_VERSION on OS X must be cpu"
+      exit 1
+    fi
+    export CPU_SUFFIX=""
+  else
+    case "$CUDA_VERSION" in
+      10.0)
+        export CUDA_SUFFIX="" ;; # default!
+      9.2)
+        export CUDA_SUFFIX="+cu92" ;;
+      cpu)
+        export CUDA_SUFFIX="+cpu" ;;
+      *)
+        echo "Unrecognized CUDA_VERSION=$CUDA_VERSION"
+    esac
+    export CPU_SUFFIX="+cpu"
+  fi
+}
+
+# If a package is cpu-only, we never provide a cuda suffix
+setup_cpuonly_cuda_suffix() {
+  export CUDA_SUFFIX=""
+  export CPU_SUFFIX=""
+}
+
+# Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string
+# Usage: setup_build_version 0.2
+setup_build_version() {
+  if [[ -z "$BUILD_VERSION" ]]; then
+    export BUILD_VERSION="$1.dev$(date "+%Y%m%d")"
+  fi
+}
+
+# Set some useful variables for OS X, if applicable
+setup_macos() {
+  if [[ "$(uname)" == Darwin ]]; then
+    export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++
+  fi
+}
+
+# Function to retry functions that sometimes timeout or have flaky failures
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+# Install with pip a bit more robustly than the default
+pip_install() {
+  retry pip install --progress-bar off "$@"
+}
+
+# Install torch with pip, respecting PYTORCH_VERSION, and record the installed
+# version into PYTORCH_VERSION, if applicable
+setup_pip_pytorch_version() {
+  if [[ -z "$PYTORCH_VERSION" ]]; then
+    # Install latest prerelease CPU version of torch, per our nightlies.
+    pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+    # CPU/CUDA variants of PyTorch have ABI compatible PyTorch.  Therefore, we
+    # strip off the local package qualifier.
+    export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version:  *//' | sed 's/+.\+//')"
+  else
+    # TODO: Maybe add staging too
+    pip_install "torch==$PYTORCH_VERSION" \
+      -f https://download.pytorch.org/whl/torch_stable.html
+  fi
+}
+
+# Fill PYTORCH_VERSION with the latest conda nightly version, and
+# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions
+#
+# You MUST have populated CUDA_SUFFIX before hand.
+#
+# TODO: This is currently hard-coded for CPU-only case
+setup_conda_pytorch_constraint() {
+  if [[ -z "$PYTORCH_VERSION" ]]; then
+    export CONDA_CHANNEL_FLAGS="-c pytorch-nightly"
+    export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")"
+  else
+    export CONDA_CHANNEL_FLAGS="-c pytorch"
+  fi
+  if [[ "$CUDA_VERSION" == cpu ]]; then
+    export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${CPU_SUFFIX}"
+    export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION"
+  else
+    export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}"
+    export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}"
+  fi
+}
+
+# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT
+setup_conda_cudatoolkit_constraint() {
+  if [[ "$CUDA_VERSION" == cpu ]]; then
+    export CONDA_CUDATOOLKIT_CONSTRAINT=""
+  else
+    echo
+    # TODO
+  fi
+}
diff --git a/build_tools/packaging/conda/torchaudio/build.sh b/packaging/torchaudio/build.sh
similarity index 64%
rename from build_tools/packaging/conda/torchaudio/build.sh
rename to packaging/torchaudio/build.sh
index 7e5917060f..dff89cf31b 100644
--- a/build_tools/packaging/conda/torchaudio/build.sh
+++ b/packaging/torchaudio/build.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -ex
 
-PREFIX="$(pwd)"
-. build_tools/packaging/wheel/build_from_source.sh
+packaging/build_from_source.sh "$(pwd)"
 
 IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt
diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml
similarity index 69%
rename from build_tools/packaging/conda/torchaudio/meta.yaml
rename to packaging/torchaudio/meta.yaml
index fdf41e52a9..0b89969fa2 100644
--- a/build_tools/packaging/conda/torchaudio/meta.yaml
+++ b/packaging/torchaudio/meta.yaml
@@ -1,9 +1,9 @@
 package:
   name: torchaudio
-  version: "{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}"
+  version: "{{ environ.get('BUILD_VERSION') }}"
 
 source:
-  path: "{{ environ.get('TORCHAUDIO_GITHUB_ROOT_DIR') }}"
+  path: "{{ environ.get('SOURCE_ROOT_DIR') }}"
 
 requirements:
   build:
@@ -13,16 +13,16 @@ requirements:
   host:
     - python
     - setuptools
-    - pytorch >=1.2.0
+    - cpuonly
+    {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }}
 
   run:
     - python
     - typing
-    - pytorch >=1.2.0
+    {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
 
 build:
-  number: {{ environ.get('TORCHAUDIO_BUILD_NUMBER') }}
-  string: py{{py}}_{{environ.get('TORCHAUDIO_BUILD_NUMBER')}}
+  string: py{{py}}
 
 test:
   imports:
diff --git a/setup.py b/setup.py
index eea67ae8ec..2d2ea43860 100644
--- a/setup.py
+++ b/setup.py
@@ -63,12 +63,8 @@ def check_env_flag(name, default=''):
 except Exception:
     pass
 
-if os.getenv('TORCHAUDIO_BUILD_VERSION'):
-    assert os.getenv('TORCHAUDIO_BUILD_NUMBER') is not None
-    build_number = int(os.getenv('TORCHAUDIO_BUILD_NUMBER'))
-    version = os.getenv('TORCHAUDIO_BUILD_VERSION')
-    if build_number > 1:
-        version += '.post' + str(build_number)
+if os.getenv('BUILD_VERSION'):
+    version = os.getenv('BUILD_VERSION')
 elif sha != 'Unknown':
     version += '+' + sha[:7]
 print('-- Building version ' + version)
@@ -78,11 +74,15 @@ def check_env_flag(name, default=''):
     f.write("__version__ = '{}'\n".format(version))
     f.write("git_version = {}\n".format(repr(sha)))
 
-pytorch_package_name = os.getenv('TORCHAUDIO_PYTORCH_DEPENDENCY_NAME', 'torch')
+pytorch_package_version = os.getenv('PYTORCH_VERSION')
+
+pytorch_package_dep = 'torch'
+if pytorch_package_version is not None:
+    pytorch_package_dep += "==" + pytorch_package_version
 
 setup(
     name="torchaudio",
-    version="0.2",
+    version=version,
     description="An audio package for PyTorch",
     url="https://github.com/pytorch/audio",
     author="Soumith Chintala, David Pollack, Sean Naren, Peter Goldsborough",