diff --git a/.github/scripts/unittest-linux/install.sh b/.github/scripts/unittest-linux/install.sh
index 885296219cd..70a8793c224 100755
--- a/.github/scripts/unittest-linux/install.sh
+++ b/.github/scripts/unittest-linux/install.sh
@@ -75,7 +75,7 @@ fi
 # Note: installing librosa via pip fail because it will try to compile numba.
 (
     set -x
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
     pip install kaldi-io SoundFile coverage pytest pytest-cov 'scipy==1.7.3' expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
 )
 # Install fairseq
diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml
index d87f6d01252..9f9c6eea356 100644
--- a/.github/workflows/build_docs.yml
+++ b/.github/workflows/build_docs.yml
@@ -75,7 +75,7 @@ jobs:
         conda install \
           --quiet --yes \
           -c conda-forge \
-          pandoc doxygen pysoundfile
+          sox libvorbis pandoc doxygen pysoundfile
         pip install --progress-bar off \
             git+https://github.com/kpu/kenlm/ flashlight-text \
             -r docs/requirements.txt -r docs/requirements-tutorials.txt
diff --git a/.github/workflows/unittest-linux-cpu.yml b/.github/workflows/unittest-linux-cpu.yml
index 50207c0131b..b96dd200cb4 100644
--- a/.github/workflows/unittest-linux-cpu.yml
+++ b/.github/workflows/unittest-linux-cpu.yml
@@ -50,6 +50,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail
diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml
index 86ab9dcd16c..d74489f90f4 100644
--- a/.github/workflows/unittest-linux-gpu.yml
+++ b/.github/workflows/unittest-linux-gpu.yml
@@ -38,6 +38,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
 
         # Set CHANNEL
         if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
@@ -76,7 +78,7 @@ jobs:
           --quiet --yes \
           -c conda-forge \
           -c numba/label/dev \
-          'librosa==0.10.0' parameterized 'requests>=2.20'
+          sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20'
         pip3 install --progress-bar off \
           kaldi-io \
           SoundFile \
diff --git a/.github/workflows/unittest-macos-cpu.yml b/.github/workflows/unittest-macos-cpu.yml
index 4e3cc598799..274d5eb648e 100644
--- a/.github/workflows/unittest-macos-cpu.yml
+++ b/.github/workflows/unittest-macos-cpu.yml
@@ -49,6 +49,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail
diff --git a/.github/workflows/unittest-windows-cpu.yml b/.github/workflows/unittest-windows-cpu.yml
index e2eb73674d7..a3ed8e34827 100644
--- a/.github/workflows/unittest-windows-cpu.yml
+++ b/.github/workflows/unittest-windows-cpu.yml
@@ -45,6 +45,7 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
 
         .github/scripts/unittest-windows/setup_env.sh
         .github/scripts/unittest-windows/install.sh
diff --git a/setup.py b/setup.py
index 0a8080d06ec..38a4ab3449d 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,6 @@
 import re
 import shutil
 import subprocess
-import sys
 from pathlib import Path
 
 import torch
@@ -83,18 +82,6 @@ def _get_packages(branch_name, tag):
     return find_packages(exclude=exclude)
 
 
-def _init_submodule():
-    print(" --- Initializing submodules")
-    try:
-        subprocess.check_call(["git", "submodule", "init"])
-        subprocess.check_call(["git", "submodule", "update"])
-    except Exception:
-        print(" --- Submodule initalization failed")
-        print("Please run:\n\tgit submodule update --init --recursive")
-        sys.exit(1)
-    print(" --- Initialized submodule")
-
-
 def _parse_url(path):
     with open(path, "r") as file_:
         for line in file_:
@@ -104,18 +91,6 @@ def _parse_url(path):
                 yield url
 
 
-def _parse_sources():
-    third_party_dir = ROOT_DIR / "third_party"
-    libs = ["sox"]
-    archive_dir = third_party_dir / "archives"
-    archive_dir.mkdir(exist_ok=True)
-    for lib in libs:
-        cmake_file = third_party_dir / lib / "CMakeLists.txt"
-        for url in _parse_url(cmake_file):
-            path = archive_dir / os.path.basename(url)
-            yield path, url
-
-
 def _fetch_archives(src):
     for dest, url in src:
         if not dest.exists():
@@ -123,13 +98,6 @@ def _fetch_archives(src):
             torch.hub.download_url_to_file(url, dest, progress=False)
 
 
-def _fetch_third_party_libraries():
-    # Revert this when a submodule is added again
-    # _init_submodule()
-    if os.name != "nt":
-        _fetch_archives(_parse_sources())
-
-
 def _main():
     sha = _run_cmd(["git", "rev-parse", "HEAD"])
     branch = _run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"])
@@ -143,7 +111,6 @@ def _main():
     print("-- Building version", version)
 
     _make_version_file(version, sha)
-    _fetch_third_party_libraries()
 
     with open("README.md") as f:
         long_description = f.read()
diff --git a/third_party/sox/CMakeLists.txt b/third_party/sox/CMakeLists.txt
index c4f5dd8931d..db96f05fafd 100644
--- a/third_party/sox/CMakeLists.txt
+++ b/third_party/sox/CMakeLists.txt
@@ -1,212 +1,21 @@
-find_package(PkgConfig REQUIRED)
+include(FetchContent)
 
-include(ExternalProject)
-
-set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install)
-set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
-set(patch_dir ${PROJECT_SOURCE_DIR}/third_party/patches)
-set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc)
-
-# To pass custom environment variables to ExternalProject_Add command,
-# we need to do `${CMAKE_COMMAND} -E env ${envs} <COMMANAD>`.
-# https://stackoverflow.com/a/62437353
-# We constrcut the custom environment variables here
-set(envs
-  "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig"
-  "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}"
-  "CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden $ENV{CFLAGS}"
-)
-
-ExternalProject_Add(amr
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz
-  URL_HASH SHA256=2c006cb9d5f651bfb5e60156dbff6af3c9d35c7bbcc9015308c0aff1e14cd341
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/amr/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/amr/configure ${COMMON_ARGS}
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-ExternalProject_Add(lame
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
-  URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/lame/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lame/configure ${COMMON_ARGS} --enable-nasm
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-ExternalProject_Add(ogg
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
-  URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/ogg/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/ogg/configure ${COMMON_ARGS}
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-ExternalProject_Add(flac
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
-  URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/flac/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/flac/configure ${COMMON_ARGS} --with-ogg --disable-cpplibs --disable-xmms-plugin
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-ExternalProject_Add(vorbis
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
-  URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/vorbis/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/vorbis/configure ${COMMON_ARGS} --with-ogg
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-ExternalProject_Add(opus
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
-  URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/opus/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/opus/configure ${COMMON_ARGS} --with-ogg
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-ExternalProject_Add(opusfile
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS opus
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
-  URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/opusfile/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-# OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses.
-# See https://github.com/pytorch/audio/pull/1026
-# TODO: Add flags like https://github.com/suphoff/pytorch_parallel_extension_cpp/blob/master/setup.py
-set(SOX_OPTIONS
-  --disable-openmp
-  --with-amrnb
-  --with-amrwb
-  --with-flac
-  --with-lame
-  --with-oggvorbis
-  --with-opus
-  --without-alsa
-  --without-ao
-  --without-coreaudio
-  --without-oss
-  --without-id3tag
-  --without-ladspa
-  --without-mad
-  --without-magic
-  --without-png
-  --without-pulseaudio
-  --without-sndfile
-  --without-sndio
-  --without-sunaudio
-  --without-waveaudio
-  --without-wavpack
-  --without-twolame
-  )
-
-set(SOX_LIBRARIES
-  ${INSTALL_DIR}/lib/libsox.a
-  ${INSTALL_DIR}/lib/libopencore-amrnb.a
-  ${INSTALL_DIR}/lib/libopencore-amrwb.a
-  ${INSTALL_DIR}/lib/libmp3lame.a
-  ${INSTALL_DIR}/lib/libFLAC.a
-  ${INSTALL_DIR}/lib/libopusfile.a
-  ${INSTALL_DIR}/lib/libopus.a
-  ${INSTALL_DIR}/lib/libvorbisenc.a
-  ${INSTALL_DIR}/lib/libvorbisfile.a
-  ${INSTALL_DIR}/lib/libvorbis.a
-  ${INSTALL_DIR}/lib/libogg.a
-  )
-
-set(sox_depends
-  ogg flac vorbis opusfile lame amr
-  )
-
-ExternalProject_Add(sox
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${sox_depends}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
+FetchContent_Declare(
+  sox_src
   URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
   URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/sox/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/sox/configure ${COMMON_ARGS} ${SOX_OPTIONS}
-  BUILD_BYPRODUCTS ${SOX_LIBRARIES}
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-add_library(libsox INTERFACE)
-add_dependencies(libsox sox)
-target_include_directories(libsox INTERFACE ${INSTALL_DIR}/include)
-target_link_libraries(libsox INTERFACE ${SOX_LIBRARIES})
+  PATCH_COMMAND ""
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  )
+# FetchContent_MakeAvailable will parse the downloaded content and setup the targets.
+# We want to only download and not build, so we run Populate manually.
+if(NOT sox_src_POPULATED)
+  FetchContent_Populate(sox_src)
+endif()
+
+add_library(sox SHARED stub.c)
+if(APPLE)
+  set_target_properties(sox PROPERTIES SUFFIX .dylib)
+endif(APPLE)
+target_include_directories(sox PUBLIC ${sox_src_SOURCE_DIR}/src)
diff --git a/third_party/sox/stub.c b/third_party/sox/stub.c
new file mode 100644
index 00000000000..4e668caf37c
--- /dev/null
+++ b/third_party/sox/stub.c
@@ -0,0 +1,85 @@
+#include <sox.h>
+
+int sox_add_effect(
+    sox_effects_chain_t* chain,
+    sox_effect_t* effp,
+    sox_signalinfo_t* in,
+    sox_signalinfo_t const* out) {
+  return -1;
+}
+int sox_close(sox_format_t* ft) {
+  return -1;
+}
+
+sox_effect_t* sox_create_effect(sox_effect_handler_t const* eh) {
+  return NULL;
+}
+
+sox_effects_chain_t* sox_create_effects_chain(
+    sox_encodinginfo_t const* in_enc,
+    sox_encodinginfo_t const* out_enc) {
+  return NULL;
+}
+
+void sox_delete_effect(sox_effect_t* effp) {}
+void sox_delete_effects_chain(sox_effects_chain_t* ecp) {}
+
+int sox_effect_options(sox_effect_t* effp, int argc, char* const argv[]) {
+  return -1;
+}
+
+const sox_effect_handler_t* sox_find_effect(char const* name) {
+  return NULL;
+}
+
+int sox_flow_effects(
+    sox_effects_chain_t* chain,
+    int callback(sox_bool all_done, void* client_data),
+    void* client_data) {
+  return -1;
+}
+
+const sox_effect_fn_t* sox_get_effect_fns(void) {
+  return NULL;
+}
+
+const sox_format_tab_t* sox_get_format_fns(void) {
+  return NULL;
+}
+
+sox_globals_t* sox_get_globals(void) {
+  return NULL;
+}
+
+sox_format_t* sox_open_read(
+    char const* path,
+    sox_signalinfo_t const* signal,
+    sox_encodinginfo_t const* encoding,
+    char const* filetype) {
+  return NULL;
+}
+
+sox_format_t* sox_open_write(
+    char const* path,
+    sox_signalinfo_t const* signal,
+    sox_encodinginfo_t const* encoding,
+    char const* filetype,
+    sox_oob_t const* oob,
+    sox_bool overwrite_permitted(char const* filename)) {
+  return NULL;
+}
+
+const char* sox_strerror(int sox_errno) {
+  return NULL;
+}
+
+size_t sox_write(sox_format_t* ft, const sox_sample_t* buf, size_t len) {
+  return 0;
+}
+
+int sox_init() {
+  return -1;
+};
+int sox_quit() {
+  return -1;
+};
diff --git a/torchaudio/_extension/__init__.py b/torchaudio/_extension/__init__.py
index b8a0e4237c1..7a51e07900d 100644
--- a/torchaudio/_extension/__init__.py
+++ b/torchaudio/_extension/__init__.py
@@ -8,7 +8,7 @@
     from .fb import _init_ffmpeg
 except ImportError:
     from .utils import _init_ffmpeg
-from .utils import _check_cuda_version, _fail_since_no_ffmpeg, _init_dll_path, _init_sox, _load_lib
+from .utils import _check_cuda_version, _fail_since_no_ffmpeg, _fail_since_no_sox, _init_dll_path, _init_sox, _load_lib
 
 _LG = logging.getLogger(__name__)
 
@@ -58,8 +58,16 @@
 # sox integration is built but libsox is not found.
 _SOX_INITIALIZED = False
 if is_module_available("torchaudio.lib._torchaudio_sox"):
-    _init_sox()
-    _SOX_INITIALIZED = True
+    try:
+        _init_sox()
+        _SOX_INITIALIZED = True
+    except Exception:
+        # The initialization of sox extension will fail if supported sox
+        # libraries are not found in the system.
+        # Since the rest of the torchaudio works without it, we do not report the
+        # error here.
+        # The error will be raised when user code attempts to use these features.
+        _LG.debug("Failed to initialize sox extension", exc_info=True)
 
 
 # Initialize FFmpeg-related features
@@ -76,14 +84,7 @@
         _LG.debug("Failed to initialize ffmpeg bindings", exc_info=True)
 
 
-fail_if_no_sox = (
-    no_op
-    if _SOX_INITIALIZED
-    else fail_with_message(
-        "requires sox extension, but TorchAudio is not compiled with it. Please build TorchAudio with libsox support."
-    )
-)
-
+fail_if_no_sox = no_op if _SOX_INITIALIZED else _fail_since_no_sox
 fail_if_no_ffmpeg = _fail_since_no_ffmpeg if _FFMPEG_EXT is None else no_op
 
 fail_if_no_rir = (
diff --git a/torchaudio/_extension/utils.py b/torchaudio/_extension/utils.py
index 2045482cb66..b12f6cc63b2 100644
--- a/torchaudio/_extension/utils.py
+++ b/torchaudio/_extension/utils.py
@@ -197,6 +197,25 @@ def _check_cuda_version():
     return version
 
 
+def _fail_since_no_sox(func):
+    @wraps(func)
+    def wrapped(*_args, **_kwargs):
+        try:
+            # Note:
+            # We run _init_ffmpeg again just to show users the stacktrace.
+            # _init_ffmpeg would not succeed here.
+            _init_sox()
+        except Exception as err:
+            raise RuntimeError(
+                f"{func.__name__} requires sox extension which is not available. "
+                "Please refer to the stacktrace above for how to resolve this."
+            ) from err
+        # This should not happen in normal execution, but just in case.
+        return func(*_args, **_kwargs)
+
+    return wrapped
+
+
 def _fail_since_no_ffmpeg(func):
     @wraps(func)
     def wrapped(*_args, **_kwargs):
diff --git a/torchaudio/csrc/sox/CMakeLists.txt b/torchaudio/csrc/sox/CMakeLists.txt
index 3391a4fc370..5ffe782c823 100644
--- a/torchaudio/csrc/sox/CMakeLists.txt
+++ b/torchaudio/csrc/sox/CMakeLists.txt
@@ -10,7 +10,7 @@ torchaudio_library(
   libtorchaudio_sox
   "${sources}"
   ""
-  "torch;libsox"
+  "torch;sox"
   ""
   )
 
diff --git a/torchaudio/csrc/sox/utils.h b/torchaudio/csrc/sox/utils.h
index 255d7270fe2..b1183659999 100644
--- a/torchaudio/csrc/sox/utils.h
+++ b/torchaudio/csrc/sox/utils.h
@@ -2,7 +2,7 @@
 #define TORCHAUDIO_SOX_UTILS_H
 
 #include <sox.h>
-#include <torch/script.h>
+#include <torch/types.h>
 
 namespace torchaudio::sox {