diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh
index 3d32e13641..a79fc73f92 100755
--- a/build_tools/packaging/conda/build_audio.sh
+++ b/build_tools/packaging/conda/build_audio.sh
@@ -10,38 +10,40 @@ retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
 }
 
-export TORCHAUDIO_BUILD_VERSION="0.2.0"
+export TORCHAUDIO_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")"
 export TORCHAUDIO_BUILD_NUMBER=1
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 audio_rootdir="$(pwd)/torchaudio-src"
 
+if [[ "$BRANCH" == "" ]]; then
+  BRANCH=master
+fi
+
 if [[ ! -d "$audio_rootdir" ]]; then
     rm -rf "$audio_rootdir"
-    git clone "https://github.com/pytorch/audio" "$audio_rootdir"
-    pushd "$audio_rootdir"
-    git checkout v$TORCHAUDIO_BUILD_VERSION
-    popd
+    git clone "https://github.com/pytorch/audio" "$audio_rootdir" -b "$BRANCH"
 fi
 
+export TORCHAUDIO_GITHUB_ROOT_DIR="$audio_rootdir"
+
 cd "$SOURCE_DIR"
 
 ANACONDA_USER=pytorch
 conda config --set anaconda_upload no
 
-# "$desired_cuda" == 'cpu'
-export TORCHAUDIO_PACKAGE_SUFFIX=""
+LATEST_PYTORCH_NIGHTLY_VERSION=$(conda search --json 'pytorch-nightly[channel=pytorch]' | python "$SOURCE_DIR/get-latest.py")
+export CONDA_PYTORCH_CONSTRAINT="    - pytorch-nightly ==${LATEST_PYTORCH_NIGHTLY_VERSION}"
 export CONDA_CUDATOOLKIT_CONSTRAINT=""
 export CUDA_VERSION="None"
-if [[ "$OSTYPE" != "darwin"* ]]; then
-    export TORCHAUDIO_PACKAGE_SUFFIX="-cpu"
-else
+if [[ "$OSTYPE" == "darwin"* ]]; then
   export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++
 fi
 
 time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio
-time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio
+# Currently disabled, see https://github.com/pytorch/pytorch/issues/23738
+# time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio
 time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio
 time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio
 
diff --git a/build_tools/packaging/conda/get-latest.py b/build_tools/packaging/conda/get-latest.py
new file mode 100644
index 0000000000..417f972f63
--- /dev/null
+++ b/build_tools/packaging/conda/get-latest.py
@@ -0,0 +1,8 @@
+# Functionally equivalent to:
+# jq -r '.["pytorch-nightly"][-1].version' | sed 's/+.*$//'
+
+import sys
+import json
+import re
+
+print(re.sub(r'\+.*$', '', json.load(sys.stdin)["pytorch-nightly"][-1]["version"]))
diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/build_tools/packaging/conda/torchaudio/meta.yaml
index 2e7d5d3944..735019e0db 100644
--- a/build_tools/packaging/conda/torchaudio/meta.yaml
+++ b/build_tools/packaging/conda/torchaudio/meta.yaml
@@ -1,10 +1,9 @@
 package:
-  name: torchaudio{{ environ.get('TORCHAUDIO_PACKAGE_SUFFIX') }}
+  name: torchaudio
   version: "{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}"
 
 source:
-  git_rev: v{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}
-  git_url: https://github.com/pytorch/audio.git
+  path: "{{ environ.get('TORCHAUDIO_GITHUB_ROOT_DIR') }}"
 
 requirements:
   build:
@@ -14,20 +13,16 @@ requirements:
   host:
     - python
     - setuptools
-    - pytorch{{ environ.get('TORCHAUDIO_PACKAGE_SUFFIX') }} >=1.1.0
-    - sox
+{{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
 
   run:
     - python
-    - pytorch{{ environ.get('TORCHAUDIO_PACKAGE_SUFFIX') }} >=1.1.0
-    - librosa >=0.4.3
-    - scipy
-    - sox
+    - typing
+{{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
 
 build:
   number: {{ environ.get('TORCHAUDIO_BUILD_NUMBER') }}
   string: py{{py}}_{{environ.get('TORCHAUDIO_BUILD_NUMBER')}}
-  script: IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt # [not win]
 
 test:
   imports:
@@ -42,7 +37,8 @@ test:
 
   requires:
     - pytest
-    - librosa >=0.4.3
+    # Ideally we would test this, but conda doesn't provide librosa
+    # - librosa >=0.4.3
     - scipy
 
 about:
diff --git a/build_tools/packaging/wheel/build_from_source.sh b/build_tools/packaging/wheel/build_from_source.sh
index 31279ac4ed..eacd890852 100755
--- a/build_tools/packaging/wheel/build_from_source.sh
+++ b/build_tools/packaging/wheel/build_from_source.sh
@@ -1,6 +1,6 @@
 rm -rf source_code
 mkdir source_code
-cd source_code
+pushd source_code
 
 wget -q -O sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet"
 wget -q -O lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent"
@@ -15,13 +15,13 @@ tar xfp libmad-0.15.1b.tar.gz
 
 # build lame, statically
 pushd lame-3.99.5
-./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/lame" CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking --enable-nasm
+./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/lame" CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking --enable-nasm
 make -s -j && make install
 popd
 
 # build flac, statically
 pushd flac-1.3.2
-./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/flac" CFLAGS=-fPIC CXXFLAGS=-fPIC \
+./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/flac" CFLAGS=-fPIC CXXFLAGS=-fPIC \
     --with-pic --disable-debug --disable-dependency-tracking
 make -s -j && make install
 popd
@@ -30,7 +30,7 @@ popd
 pushd libmad-0.15.1b
 # See https://stackoverflow.com/a/12864879/23845
 sed -i.bak 's/-march=i486//' configure
-./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/mad" CFLAGS=-fPIC CXXFLAGS=-fPIC \
+./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/mad" CFLAGS=-fPIC CXXFLAGS=-fPIC \
     --with-pic --disable-debug --disable-dependency-tracking
 make -s -j && make install
 popd
@@ -40,9 +40,11 @@ popd
 # finds png and enables it.  We don't want it; we'd need to package
 # it statically if we do.
 pushd sox-14.4.2
-./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/sox" \
-    LDFLAGS="-L$PREFIX/audio/third_party/lame/lib -L$PREFIX/audio/third_party/flac/lib -L$PREFIX/audio/third_party/mad/lib" \
-    CPPFLAGS="-I$PREFIX/audio/third_party/lame/include -I$PREFIX/audio/third_party/flac/include -I$PREFIX/audio/third_party/mad/include" \
+./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/sox" \
+    LDFLAGS="-L$PREFIX/third_party/lame/lib -L$PREFIX/third_party/flac/lib -L$PREFIX/third_party/mad/lib" \
+    CPPFLAGS="-I$PREFIX/third_party/lame/include -I$PREFIX/third_party/flac/include -I$PREFIX/third_party/mad/include" \
     --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
 make -s -j && make install
 popd
+
+popd
diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh
index 519784099c..3662ddcf9e 100755
--- a/build_tools/packaging/wheel/linux_manywheel.sh
+++ b/build_tools/packaging/wheel/linux_manywheel.sh
@@ -19,7 +19,7 @@ rm -rf audio
 git clone https://github.com/pytorch/audio
 mkdir audio/third_party
 
-export PREFIX="/tmp"
+export PREFIX="/tmp/audio"
 . /remote/wheel/build_from_source.sh
 
 cd /tmp/audio
diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh
index a791888381..fdcb18e223 100755
--- a/build_tools/packaging/wheel/osx_wheel.sh
+++ b/build_tools/packaging/wheel/osx_wheel.sh
@@ -29,7 +29,7 @@ rm -rf audio
 git clone https://github.com/pytorch/audio
 mkdir audio/third_party
 
-export PREFIX="/tmp"
+export PREFIX="/tmp/audio"
 . $CURR_PATH/build_from_source.sh
 
 cd /tmp/audio
diff --git a/setup.py b/setup.py
index a96b6c6247..0b8d0940ff 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,10 @@ def check_env_flag(name, default=''):
 include_dirs = []
 extra_objects = []
 
-if IS_WHEEL:
+# Hypothetically, the conda distribution could rely on an external sox,
+# but the library is pretty small and it is not available on the default
+# anaconda channel.  So we statically link it in, just as we do with wheels.
+if IS_WHEEL or IS_CONDA:
     audio_path = os.path.dirname(os.path.abspath(__file__))
 
     include_dirs += [os.path.join(audio_path, 'third_party/flac/include')]
@@ -49,11 +52,6 @@ def check_env_flag(name, default=''):
 else:
     libraries += ['sox']
 
-if IS_CONDA:
-    # We want $PREFIX/include for conda (for sox.h)
-    lib_path = os.path.dirname(sys.executable)
-    include_dirs += [os.path.join(os.path.dirname(lib_path), 'include')]
-
 
 # Creating the version file
 cwd = os.path.dirname(os.path.abspath(__file__))