Temporarily skip AIO tests due to an issue with runners (deepspeedai#7288)

Signed-off-by: Logan Adams <loadams@microsoft.com>
Signed-off-by: Olatunji Ruwase <tunji.ruwase@snowflake.com>
Co-authored-by: Olatunji Ruwase <tunji.ruwase@snowflake.com>
Signed-off-by: Max Kovalenko <mkovalenko@habana.ai>

Author: 2 people

.github/workflows/setup-venv/action.yml

@@ -6,7 +6,9 @@ runs:
     - id: update-env
       run: |
         sudo apt-get update
-        sudo apt-get install -y libaio-dev
+        # Temporary disable nvme UTs
+        # sudo apt-get install -y libaio-dev
+        sudo apt remove -y libaio-dev
         python -m pip install --user --upgrade pip
         python -m pip install --user --upgrade virtualenv
       shell: bash

tests/unit/inference/quantization/test_intX_quantization.py

@@ -17,6 +17,7 @@
 import pytest
 from collections import OrderedDict
 from typing import Dict
+from deepspeed.ops.aio import AsyncIOBuilder
 
 device = get_accelerator().device_name() if get_accelerator().is_available() else 'cpu'
 
@@ -57,6 +58,9 @@ def zero3_post_init_quantization_test_helper(cpu_offload: bool, nvme_offload: bo
     import deepspeed
     from transformers.integrations.deepspeed import HfDeepSpeedConfig
 
+    if nvme_offload and not deepspeed.ops.__compatible_ops__[AsyncIOBuilder.NAME]:
+        pytest.skip('Skip tests since async-io is not compatible')
+
     def get_zero3_ds_config(hf_config: OPTConfig, cpu_offload: bool, nvme_offload: bool, bits: int) -> Dict:
         GB = 1 << 30
 
@@ -174,6 +178,9 @@ def zero3_quantized_initialization_test_helper(cpu_offload: bool, nvme_offload:
     import deepspeed
     from transformers.integrations.deepspeed import HfDeepSpeedConfig
 
+    if nvme_offload and not deepspeed.ops.__compatible_ops__[AsyncIOBuilder.NAME]:
+        pytest.skip('Skip tests since async-io is not compatible')
+
     def get_zero3_ds_config(hf_config: OPTConfig, cpu_offload: bool, nvme_offload: bool, bits: int) -> Dict:
         GB = 1 << 30
 

tests/unit/runtime/compile/test_compile_zero.py

@@ -13,6 +13,8 @@
 from unit.runtime.compile.util import compare_loss
 from unit.common import DistributedTest
 from unit.util import bf16_required_version_check, skip_on_arch
+import deepspeed
+from deepspeed.ops.aio import AsyncIOBuilder
 
 pytestmark = pytest.mark.skipif(not required_torch_version(min_version=2.1),
                                 reason="Compile tests requires Pytorch version 2.1 or above")
@@ -36,6 +38,8 @@ def test_compile_zero(self, tmpdir, zero_stage, dtype, offload_device):
             pytest.skip("CPU does not support this test yet")
 
         if offload_device == OffloadDeviceEnum.nvme:
+            if not deepspeed.ops.__compatible_ops__[AsyncIOBuilder.NAME]:
+                pytest.skip('Skip tests since async-io is not compatible')
             if zero_stage != 3:
                 pytest.skip(f"Nvme offload not supported for zero stage {zero_stage}")