diff --git a/.github/workflows/unittest-linux-cpu.yml b/.github/workflows/unittest-linux-cpu.yml
index 6deb5412d0c..b96dd200cb4 100644
--- a/.github/workflows/unittest-linux-cpu.yml
+++ b/.github/workflows/unittest-linux-cpu.yml
@@ -49,6 +49,9 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail
diff --git a/.github/workflows/unittest-macos-cpu.yml b/.github/workflows/unittest-macos-cpu.yml
index 1324f1dbabc..274d5eb648e 100644
--- a/.github/workflows/unittest-macos-cpu.yml
+++ b/.github/workflows/unittest-macos-cpu.yml
@@ -48,6 +48,9 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail
diff --git a/.github/workflows/unittest-windows-cpu.yml b/.github/workflows/unittest-windows-cpu.yml
index e2eb73674d7..a3ed8e34827 100644
--- a/.github/workflows/unittest-windows-cpu.yml
+++ b/.github/workflows/unittest-windows-cpu.yml
@@ -45,6 +45,7 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
 
         .github/scripts/unittest-windows/setup_env.sh
         .github/scripts/unittest-windows/install.sh
diff --git a/test/torchaudio_unittest/backend/dispatcher/ffmpeg/load_test.py b/test/torchaudio_unittest/backend/dispatcher/ffmpeg/load_test.py
index 4a5689df4ad..ee15a95c616 100644
--- a/test/torchaudio_unittest/backend/dispatcher/ffmpeg/load_test.py
+++ b/test/torchaudio_unittest/backend/dispatcher/ffmpeg/load_test.py
@@ -11,6 +11,7 @@
 
 from torchaudio_unittest.backend.dispatcher.sox.common import name_func
 from torchaudio_unittest.common_utils import (
+    disabledInCI,
     get_asset_path,
     get_wav_data,
     HttpServerMixin,
@@ -535,6 +536,7 @@ def read(self, n):
         return self.fileobj.read(n)
 
 
+@disabledInCI
 @skipIfNoFFmpeg
 @skipIfNoExec("sox")
 @skipIfNoModule("requests")
diff --git a/test/torchaudio_unittest/backend/dispatcher/ffmpeg/save_test.py b/test/torchaudio_unittest/backend/dispatcher/ffmpeg/save_test.py
index 98120f2f4fd..42865470f51 100644
--- a/test/torchaudio_unittest/backend/dispatcher/ffmpeg/save_test.py
+++ b/test/torchaudio_unittest/backend/dispatcher/ffmpeg/save_test.py
@@ -12,6 +12,7 @@
 
 from torchaudio_unittest.backend.dispatcher.sox.common import get_enc_params, name_func
 from torchaudio_unittest.common_utils import (
+    disabledInCI,
     get_wav_data,
     load_wav,
     nested_params,
@@ -147,6 +148,7 @@ def assert_save_consistency(
         self.assertEqual(found, expected)
 
 
+@disabledInCI
 @skipIfNoExec("sox")
 @skipIfNoExec("ffmpeg")
 @skipIfNoFFmpeg
@@ -396,6 +398,7 @@ def test_save_tensor_preserve(self, dtype):
         self.assertEqual(data, expected)
 
 
+@disabledInCI
 @skipIfNoExec("sox")
 @skipIfNoFFmpeg
 class TestSaveNonExistingDirectory(PytorchTestCase):
diff --git a/test/torchaudio_unittest/backend/dispatcher/sox/info_test.py b/test/torchaudio_unittest/backend/dispatcher/sox/info_test.py
index 7d87e375ee6..2c206895f1d 100644
--- a/test/torchaudio_unittest/backend/dispatcher/sox/info_test.py
+++ b/test/torchaudio_unittest/backend/dispatcher/sox/info_test.py
@@ -7,6 +7,7 @@
 from torchaudio._internal import module_utils as _mod_utils
 from torchaudio_unittest.backend.common import get_encoding
 from torchaudio_unittest.common_utils import (
+    disabledInCI,
     get_asset_path,
     get_wav_data,
     HttpServerMixin,
@@ -15,6 +16,7 @@
     skipIfNoExec,
     skipIfNoModule,
     skipIfNoSox,
+    skipIfNoSoxDecoder,
     sox_utils,
     TempDirMixin,
 )
@@ -178,6 +180,7 @@ def test_amb(self, dtype, sample_rate, num_channels):
         assert info.bits_per_sample == bits_per_sample
         assert info.encoding == get_encoding("amb", dtype)
 
+    @skipIfNoSoxDecoder("amr_nb")
     def test_amr_nb(self):
         """`self._info` can check amr-nb file correctly"""
         duration = 1
@@ -256,7 +259,8 @@ def test_htk(self):
         assert info.encoding == "PCM_S"
 
 
-@skipIfNoSox
+@disabledInCI
+@skipIfNoSoxDecoder("opus")
 class TestInfoOpus(PytorchTestCase):
     _info = partial(get_info_func(), backend="sox")
 
diff --git a/test/torchaudio_unittest/backend/dispatcher/sox/load_test.py b/test/torchaudio_unittest/backend/dispatcher/sox/load_test.py
index 58c6098a2c6..fa9654540b5 100644
--- a/test/torchaudio_unittest/backend/dispatcher/sox/load_test.py
+++ b/test/torchaudio_unittest/backend/dispatcher/sox/load_test.py
@@ -13,6 +13,7 @@
     save_wav,
     skipIfNoExec,
     skipIfNoSox,
+    skipIfNoSoxDecoder,
     sox_utils,
     TempDirMixin,
 )
@@ -237,6 +238,7 @@ def test_vorbis_large(self, sample_rate, num_channels, quality_level):
         ),
         name_func=name_func,
     )
+    @skipIfNoSoxDecoder("opus")
     def test_opus(self, bitrate, num_channels, compression_level):
         """`sox_io_backend.load` can load opus file correctly."""
         ops_path = get_asset_path("io", f"{bitrate}_{compression_level}_{num_channels}ch.opus")
@@ -281,6 +283,7 @@ def test_amb(self, dtype, sample_rate, num_channels, normalize):
             "amb", sample_rate, num_channels, bit_depth=bit_depth, duration=1, encoding=encoding, normalize=normalize
         )
 
+    @skipIfNoSoxDecoder("amr_nb")
     def test_amr_nb(self):
         """`sox_io_backend.load` can load amr_nb format correctly."""
         self.assert_format("amr-nb", sample_rate=8000, num_channels=1, bit_depth=32, duration=1)
diff --git a/test/torchaudio_unittest/backend/dispatcher/sox/save_test.py b/test/torchaudio_unittest/backend/dispatcher/sox/save_test.py
index eec476b6d24..afa63f69a1c 100644
--- a/test/torchaudio_unittest/backend/dispatcher/sox/save_test.py
+++ b/test/torchaudio_unittest/backend/dispatcher/sox/save_test.py
@@ -13,6 +13,7 @@
     save_wav,
     skipIfNoExec,
     skipIfNoSox,
+    skipIfNoSoxEncoder,
     sox_utils,
     TempDirMixin,
     TorchaudioTestCase,
@@ -253,6 +254,7 @@ def test_save_amb(self, enc_params):
         encoding, bits_per_sample = enc_params
         self.assert_save_consistency("amb", encoding=encoding, bits_per_sample=bits_per_sample, test_mode="path")
 
+    @skipIfNoSoxEncoder("amr_nb")
     def test_save_amr_nb(self):
         self.assert_save_consistency("amr-nb", num_channels=1, test_mode="path")
 
@@ -269,12 +271,18 @@ def test_save_gsm(self):
             ("flac",),
             ("vorbis",),
             ("sph", "PCM_S", 16),
-            ("amr-nb",),
             ("amb", "PCM_S", 16),
         ],
         name_func=name_func,
     )
     def test_save_large(self, format, encoding=None, bits_per_sample=None):
+        self._test_save_large(format, encoding, bits_per_sample)
+
+    @skipIfNoSoxEncoder("amr_nb")
+    def test_save_large_amr_nb(self):
+        self._test_save_large("amr_nb")
+
+    def _test_save_large(self, format, encoding=None, bits_per_sample=None):
         """`self._save` can save large files."""
         sample_rate = 8000
         one_hour = 60 * 60 * sample_rate
diff --git a/test/torchaudio_unittest/backend/sox_io/info_test.py b/test/torchaudio_unittest/backend/sox_io/info_test.py
index eea03133142..67782091cc8 100644
--- a/test/torchaudio_unittest/backend/sox_io/info_test.py
+++ b/test/torchaudio_unittest/backend/sox_io/info_test.py
@@ -10,6 +10,7 @@
     save_wav,
     skipIfNoExec,
     skipIfNoSox,
+    skipIfNoSoxDecoder,
     sox_utils,
     TempDirMixin,
 )
@@ -196,6 +197,7 @@ def test_amb(self, dtype, sample_rate, num_channels):
         assert info.bits_per_sample == bits_per_sample
         assert info.encoding == get_encoding("amb", dtype)
 
+    @skipIfNoSoxDecoder("amr_nb")
     def test_amr_nb(self):
         """`sox_io_backend.info` can check amr-nb file correctly"""
         duration = 1
diff --git a/test/torchaudio_unittest/backend/sox_io/load_test.py b/test/torchaudio_unittest/backend/sox_io/load_test.py
index ea168030777..2996885bd61 100644
--- a/test/torchaudio_unittest/backend/sox_io/load_test.py
+++ b/test/torchaudio_unittest/backend/sox_io/load_test.py
@@ -12,6 +12,7 @@
     save_wav,
     skipIfNoExec,
     skipIfNoSox,
+    skipIfNoSoxDecoder,
     sox_utils,
     TempDirMixin,
 )
@@ -234,6 +235,7 @@ def test_vorbis_large(self, sample_rate, num_channels, quality_level):
         ),
         name_func=name_func,
     )
+    @skipIfNoSoxDecoder("opus")
     def test_opus(self, bitrate, num_channels, compression_level):
         """`sox_io_backend.load` can load opus file correctly."""
         ops_path = get_asset_path("io", f"{bitrate}_{compression_level}_{num_channels}ch.opus")
@@ -278,6 +280,7 @@ def test_amb(self, dtype, sample_rate, num_channels, normalize):
             "amb", sample_rate, num_channels, bit_depth=bit_depth, duration=1, encoding=encoding, normalize=normalize
         )
 
+    @skipIfNoSoxDecoder("amr_nb")
     def test_amr_nb(self):
         """`sox_io_backend.load` can load amr_nb format correctly."""
         self.assert_format("amr-nb", sample_rate=8000, num_channels=1, bit_depth=32, duration=1)
diff --git a/test/torchaudio_unittest/backend/sox_io/save_test.py b/test/torchaudio_unittest/backend/sox_io/save_test.py
index a92b3d98cb6..1057a30ca68 100644
--- a/test/torchaudio_unittest/backend/sox_io/save_test.py
+++ b/test/torchaudio_unittest/backend/sox_io/save_test.py
@@ -11,6 +11,7 @@
     save_wav,
     skipIfNoExec,
     skipIfNoSox,
+    skipIfNoSoxEncoder,
     sox_utils,
     TempDirMixin,
     TorchaudioTestCase,
@@ -265,6 +266,7 @@ def test_save_amb(self, enc_params):
             7,
         ],
     )
+    @skipIfNoSoxEncoder("amr-nb")
     def test_save_amr_nb(self, bit_rate):
         self.assert_save_consistency("amr-nb", compression=bit_rate, num_channels=1)
 
@@ -281,12 +283,18 @@ def test_save_gsm(self):
             ("flac",),
             ("vorbis",),
             ("sph", "PCM_S", 16),
-            ("amr-nb",),
             ("amb", "PCM_S", 16),
         ],
         name_func=name_func,
     )
     def test_save_large(self, format, encoding=None, bits_per_sample=None):
+        self._test_save_large(format, encoding, bits_per_sample)
+
+    @skipIfNoSoxEncoder("amr-nb")
+    def test_save_large_amr_nb(self):
+        self.test_save_large("amr_nb")
+
+    def _test_save_large(self, format, encoding=None, bits_per_sample=None):
         """`sox_io_backend.save` can save large files."""
         sample_rate = 8000
         one_hour = 60 * 60 * sample_rate
diff --git a/test/torchaudio_unittest/common_utils/__init__.py b/test/torchaudio_unittest/common_utils/__init__.py
index 8c66f2149a4..bb3f813230f 100644
--- a/test/torchaudio_unittest/common_utils/__init__.py
+++ b/test/torchaudio_unittest/common_utils/__init__.py
@@ -18,6 +18,8 @@
     skipIfNoQengine,
     skipIfNoRIR,
     skipIfNoSox,
+    skipIfNoSoxDecoder,
+    skipIfNoSoxEncoder,
     skipIfPy310,
     skipIfRocm,
     TempDirMixin,
@@ -53,7 +55,8 @@
     "skipIfNoModule",
     "skipIfNoRIR",
     "skipIfNoSox",
-    "skipIfNoSoxBackend",
+    "skipIfNoSoxDecoder",
+    "skipIfNoSoxEncoder",
     "skipIfRocm",
     "skipIfNoQengine",
     "skipIfNoFFmpeg",
diff --git a/test/torchaudio_unittest/common_utils/case_utils.py b/test/torchaudio_unittest/common_utils/case_utils.py
index a094f5d64f0..ad760afa53b 100644
--- a/test/torchaudio_unittest/common_utils/case_utils.py
+++ b/test/torchaudio_unittest/common_utils/case_utils.py
@@ -234,6 +234,24 @@ def skipIfNoModule(module, display_name=None):
     reason="Sox features are not available.",
     key="NO_SOX",
 )
+
+
+def skipIfNoSoxDecoder(ext):
+    return _skipIf(
+        not torchaudio._extension._SOX_INITIALIZED or ext not in torchaudio.utils.sox_utils.list_read_formats(),
+        f'sox does not handle "{ext}" for read.',
+        key="NO_SOX_DECODER",
+    )
+
+
+def skipIfNoSoxEncoder(ext):
+    return _skipIf(
+        not torchaudio._extension._SOX_INITIALIZED or ext not in torchaudio.utils.sox_utils.list_write_formats(),
+        f'sox does not handle "{ext}" for write.',
+        key="NO_SOX_ENCODER",
+    )
+
+
 skipIfNoRIR = _skipIf(
     not torchaudio._extension._IS_RIR_AVAILABLE,
     reason="RIR features are not available.",
diff --git a/test/torchaudio_unittest/sox_effect/dataset_test.py b/test/torchaudio_unittest/sox_effect/dataset_test.py
index ba3b6edf8e0..2f8c108e04e 100644
--- a/test/torchaudio_unittest/sox_effect/dataset_test.py
+++ b/test/torchaudio_unittest/sox_effect/dataset_test.py
@@ -94,8 +94,9 @@ def test_apply_effects_file(self):
         loader = torch.utils.data.DataLoader(
             dataset,
             batch_size=32,
-            num_workers=16,
+            num_workers=4,
             worker_init_fn=init_random_seed,
+            multiprocessing_context=torch.multiprocessing.get_context("spawn"),
         )
         for batch in loader:
             assert batch.shape == (32, 2, 2 * sample_rate)
@@ -115,8 +116,9 @@ def test_apply_effects_tensor(self):
         loader = torch.utils.data.DataLoader(
             dataset,
             batch_size=32,
-            num_workers=16,
+            num_workers=4,
             worker_init_fn=init_random_seed,
+            multiprocessing_context=torch.multiprocessing.get_context("spawn"),
         )
         for batch in loader:
             assert batch.shape == (32, 2, 2 * sample_rate)