Merge branch 'main' into no_lfilter_generic_loop

Author: samanklesaria

.github/scripts/unittest-linux/install.sh

@@ -24,41 +24,7 @@ esac
 conda create -n ci -y python="${PYTHON_VERSION}"
 conda activate ci
 
-# 1. Install PyTorch
-# if [ -z "${CUDA_VERSION:-}" ] ; then
-#     if [ "${os}" == MacOSX ] ; then
-#         cudatoolkit=''
-#     else
-#         cudatoolkit="cpuonly"
-#     fi
-#     version="cpu"
-# else
-#     version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
-#     export CUDATOOLKIT_CHANNEL="nvidia"
-#     cudatoolkit="pytorch-cuda=${version}"
-# fi
-
-# printf "Installing PyTorch with %s\n" "${cudatoolkit}"
-# (
-#     if [ "${os}" == MacOSX ] ; then
-#       # TODO: this can be removed as soon as linking issue could be resolved
-#       #  see https://github.com/pytorch/pytorch/issues/62424 from details
-#       MKL_CONSTRAINT='mkl==2021.2.0'
-#       pytorch_build=pytorch
-#     else
-#       MKL_CONSTRAINT=''
-#       pytorch_build="pytorch[build="*${version}*"]"
-#     fi
-#     set -x
-
-#     if [[ -z "$cudatoolkit" ]]; then
-#         conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" $MKL_CONSTRAINT "pytorch-${UPLOAD_CHANNEL}::${pytorch_build}"
-#     else
-#         conda install pytorch ${cudatoolkit} ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia  $MKL_CONSTRAINT
-#     fi
-# )
-
-export GPU_ARCH_TYPE="cpu"  # TODO change this
+export GPU_ARCH_TYPE="cpu"
 
 case $GPU_ARCH_TYPE in
   cpu)
@@ -74,7 +40,7 @@ case $GPU_ARCH_TYPE in
     ;;
 esac
 PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${GPU_ARCH_ID}"
-pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
+pip install --progress-bar=off --pre torch --index-url="${PYTORCH_WHEEL_INDEX}"
 
 
 # 2. Install torchaudio
@@ -88,24 +54,5 @@ pip install . -v --no-build-isolation
 printf "* Installing test tools\n"
 # On this CI, for whatever reason, we're only able to install ffmpeg 4.
 conda install -y "ffmpeg<5"
-python -c "import torch; import torchaudio; import torchcodec; print(torch.__version__, torchaudio.__version__, torchcodec.__version__)"
-
-NUMBA_DEV_CHANNEL=""
-if [[ "$(python --version)" = *3.9* || "$(python --version)" = *3.10* ]]; then
-    # Numba isn't available for Python 3.9 and 3.10 except on the numba dev channel and building from source fails
-    # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048
-    NUMBA_DEV_CHANNEL="-c numba/label/dev"
-fi
-(
-    set -x
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis parameterized 'requests>=2.20'
-    pip install kaldi-io SoundFile librosa coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
 
-    # TODO: might be better to fix the single call to `pip install` above
-    pip install "pillow<10.0" "scipy<1.10" "numpy<2.0"
-)
-# Install fairseq
-git clone https://github.com/pytorch/fairseq
-cd fairseq
-git checkout e47a4c8
-pip install .
+pip3 install parameterized requests coverage pytest pytest-cov scipy numpy expecttest

.github/scripts/unittest-linux/run_test.sh

@@ -29,6 +29,10 @@ fi
 )
 
 (
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER=true
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_unidecode=true
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_inflect=true
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_pytorch_lightning=true
     cd test
-    pytest torchaudio_unittest -k "not backend and not /io/ and not prototype and not sox and not ffmpeg and not fairseq and not hdemucs and not (torchscript and rnnt) and not torchscript_consistency"
+    pytest torchaudio_unittest -k "not torchscript and not fairseq and not demucs ${PYTEST_K_EXTRA}" 
 )

.github/scripts/unittest-windows/environment.yml

@@ -8,8 +8,6 @@ dependencies:
   - scipy >= 1.4.1
   - pip
   - pip:
-    - kaldi-io
-    - PySoundFile
     - future
     - parameterized
     - dataclasses

.github/scripts/unittest-windows/install.sh

@@ -20,16 +20,17 @@ conda activate "${env_dir}"
 
 source "$this_dir/set_cuda_envs.sh"
 
+printf "* Installing torch import-time dependencies\n"
+pip install numpy
+
 # 1. Install PyTorch
 if [ -z "${CUDA_VERSION:-}" ] ; then
-    cudatoolkit="cpuonly"
-    version="cpu"
+    wheel="cpu"
 else
-    version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
-    cudatoolkit="pytorch-cuda=${version}"
+    wheel="cu$(python -c "print(''.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
 fi
-printf "Installing PyTorch with %s\n" "${cudatoolkit}"
-conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia pytorch "${cudatoolkit}"  pytest pybind11
+printf "Installing PyTorch\n"
+pip install --pre torch --index-url https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${wheel}
 
 torch_cuda=$(python -c "import torch; print(torch.cuda.is_available())")
 echo torch.cuda.is_available is $torch_cuda
@@ -42,31 +43,17 @@ if [ ! -z "${CUDA_VERSION:-}" ] ; then
 fi
 
 # 2. Install torchaudio
-printf "* Installing fsspec\n"
+printf "* Installing fsspec\n"   # TODO: is this required for torchaudio??
 pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org fsspec
 
 printf "* Installing torchaudio\n"
 "$root_dir/packaging/vc_env_helper.bat" pip install . -v --no-build-isolation
 
 # 3. Install Test tools
 printf "* Installing test tools\n"
-NUMBA_DEV_CHANNEL=""
 SENTENCEPIECE_DEPENDENCY="sentencepiece"
-case "$(python --version)" in
-    *3.9*)
-        # Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails
-        # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048
-        NUMBA_DEV_CHANNEL="-c numba/label/dev"
-        ;;
-    *3.10*)
-        # Don't install sentencepiece, no python 3.10 dependencies available for windows yet
-        SENTENCEPIECE_DEPENDENCY=""
-        NUMBA_DEV_CHANNEL="-c numba/label/dev"
-        ;;
-esac
-# Note: installing librosa via pip fail because it will try to compile numba.
 (
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa==0.10.0' parameterized 'requests>=2.20'
+    conda install -y -c conda-forge parameterized 'requests>=2.20'
     # Need to disable shell check since this'll fail out if SENTENCEPIECE_DEPENDENCY is empty
     # shellcheck disable=SC2086
     pip install \
@@ -76,21 +63,7 @@ esac
         coverage \
         expecttest \
         inflect \
-        kaldi-io \
         pytest \
         pytest-cov \
-        pytorch-lightning \
-        'scipy==1.7.3' \
-        unidecode \
-        'protobuf<4.21.0' \
-        demucs \
-        tinytag \
-        pyroomacoustics \
-        flashlight-text \
-        git+https://github.com/kpu/kenlm/
+        scipy
 )
-# Install fairseq
-git clone https://github.com/pytorch/fairseq
-cd fairseq
-git checkout e47a4c8
-pip install .

.github/scripts/unittest-windows/run_test.sh

@@ -12,5 +12,5 @@ python -m torch.utils.collect_env
 env | grep TORCHAUDIO || true
 
 cd test
-pytest --continue-on-collection-errors --cov=torchaudio --junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml -v --durations 20 torchaudio_unittest
+pytest --continue-on-collection-errors --cov=torchaudio --junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml -v --durations 20 torchaudio_unittest -k "not torchscript and not fairseq and not demucs and not librosa"
 coverage html

.github/scripts/unittest-windows/setup_env.sh

@@ -19,7 +19,9 @@ if [ ! -d "${conda_dir}" ]; then
     printf "* Installing conda\n"
     export tmp_conda="$(echo $conda_dir | tr '/' '\\')"
     export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe"
-    curl --silent --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
+    # not using miniconda because its installation will be incomplete
+    # despite using `start /wait` in the install batch file
+    curl -L --silent --output miniconda.exe https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe -O
     "$this_dir/install_conda.bat"
     unset tmp_conda
     unset miniconda_exe
@@ -34,5 +36,4 @@ fi
 conda activate "${env_dir}"
 
 # 3. Install minimal build tools
-pip --quiet install cmake ninja
-conda install --quiet -y 'ffmpeg>=4.1'
+conda install -y -c conda-forge cmake ninja

.github/workflows/build_docs.yml

@@ -59,35 +59,25 @@ jobs:
 
         echo "::endgroup::"
         echo "::group::Install PyTorch"
-        # conda install \
-        #   --yes \
-        #   --quiet \
-        #   -c "pytorch-${CHANNEL}" \
-        #   -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${CU_VERSION}*"] \
-        #   "${CUDATOOLKIT}"
 
         GPU_ARCH_ID=cu126  # This is hard-coded and must be consistent with gpu-arch-version.
         PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}"
-        pip install --progress-bar=off --pre torch --index-url="${PYTORCH_WHEEL_INDEX}"
+        pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
 
         echo "::endgroup::"
         echo "::group::Install TorchAudio"
         conda install --quiet --yes cmake>=3.18.0 ninja
         pip3 install --progress-bar off -v -e . --no-use-pep517
 
+        # TODO: Need to rely on torchcodec instead of building ffmpeg from source.
         echo "::endgroup::"
         echo "::group::Build FFmpeg"
         .github/scripts/ffmpeg/build_gpu.sh
 
         echo "::endgroup::"
         echo "::group::Install other dependencies"
-        conda install \
-          --quiet --yes \
-          -c conda-forge \
-          sox libvorbis pandoc doxygen pysoundfile
-        pip install --progress-bar off \
-            git+https://github.com/kpu/kenlm/ flashlight-text \
-            -r docs/requirements.txt -r docs/requirements-tutorials.txt
+
+        pip install --progress-bar off -r docs/requirements.txt -r docs/requirements-tutorials.txt
 
         echo "::endgroup::"
         echo "::group::Build documentation"

.github/workflows/unittest-linux-cpu.yml

@@ -31,6 +31,8 @@ jobs:
 
         # Set up Environment Variables
         export PYTHON_VERSION="${{ matrix.python_version }}"
+        export PIP_PROGRESS_BAR=off
+        export CONDA_QUIET=1
         export CUDA_TESTS_ONLY="0"
         unset CUDA_VERSION
 

.github/workflows/unittest-linux-gpu.yml

@@ -32,19 +32,17 @@ jobs:
         set -ex
         # Set up Environment Variables
         export PYTHON_VERSION="${{ matrix.python_version }}"
+        export PIP_PROGRESS_BAR=off
+        export CONDA_QUIET=1
         export CU_VERSION="${{ matrix.cuda_arch_version }}"
         export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true
         # Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility
         export CUBLAS_WORKSPACE_CONFIG=:4096:8
 
@@ -78,35 +76,8 @@ jobs:
 
         echo "::endgroup::"
         echo "::group::Install other Dependencies"
-        # conda install \
-        #   --quiet --yes \
-        #   -c conda-forge \
-        #   -c numba/label/dev \
-        #   sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20'
-        # pip3 install --progress-bar off \
-        #   kaldi-io \
-        #   SoundFile \
-        #   coverage \
-        #   pytest \
-        #   pytest-cov \
-        #   scipy \
-        #   transformers \
-        #   expecttest \
-        #   unidecode \
-        #   inflect \
-        #   Pillow \
-        #   sentencepiece \
-        #   pytorch-lightning \
-        #   'protobuf<4.21.0' \
-        #   demucs \
-        #   tinytag \
-        #   flashlight-text \
-        #   git+https://github.com/kpu/kenlm/ \
-        #   git+https://github.com/pytorch/fairseq.git@e47a4c8
         
-        pip3 install parameterized requests
-        pip3 install kaldi-io SoundFile librosa coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag 
-        pip3 install "pillow<10.0" "scipy<1.10" "numpy<2.0"
+        pip3 install parameterized requests coverage pytest pytest-cov scipy numpy expecttest 
 
         echo "::endgroup::"
         echo "::group::Run tests"